[ May 1 00:16:40 Enabled. ] [ May 1 00:16:40 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ May 1 00:16:40 Method "start" exited with status 0. ] [ May 1 00:16:40 Rereading configuration. ] [ May 1 00:16:40 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:40.373796752Z","hostname":"buskin","pid":1590,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:40.373953774Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:40.429526825Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:40.429572616Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:40.545325869Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:40.555033059Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:40.563584126Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.169805636Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.312722497Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.312755888Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.312824899Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.312915809Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.312922549Z","hostname":"buskin","pid":1590,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.31294226Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.31295606Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.315823133Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:41.315835933Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.315853953Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.315859963Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.315894273Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.315948334Z","hostname":"buskin","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.316020335Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.316041975Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.316062255Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.328899551Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.328941011Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.454381845Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.692649979Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.692699999Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.692709709Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.692730999Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.69274513Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.69277485Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.69278361Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.701620011Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.701650281Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"49112153-160e-48f4-abfa-c3642e83cda8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:41.803849871Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.029761223Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.029805404Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.029861255Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.029868525Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.029897655Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.029925105Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.029940165Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.029947625Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.029995846Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030024497Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030052117Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030056147Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030061487Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030065097Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030079247Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030083197Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030088487Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030096597Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030100387Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030105567Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030120167Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030140028Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030143838Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030152078Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030168058Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030175998Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030180848Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030185218Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030202459Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030225759Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030238779Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030256659Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030262349Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030269099Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030272779Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030278429Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.03030391Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.03031594Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.03031969Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.03032287Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.03034459Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030399561Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030413801Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030422961Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030439671Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030447591Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030464682Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030479042Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.030488552Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.041077922Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.041118192Z","hostname":"buskin","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999929859s","next_cleanup":"Instant { tv_sec: 1942, tv_nsec: 968947188 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.041173443Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.041189163Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.041242184Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.041251904Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.044583632Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.044652463Z","hostname":"buskin","pid":1590,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.044912545Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:979"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.044918975Z","hostname":"buskin","pid":1590,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.044976426Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045026087Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045054747Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045067057Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045103697Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045124778Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045143208Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045150978Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1080"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045155088Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045162178Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045180109Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045211409Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045558823Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045779575Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045790335Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045835406Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.045928047Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046077159Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.04622023Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046299351Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 9f9b1e33-4da8-44e7-8e80-581afd22135c, rack_id: 3f09cc19-1113-4b10-8b38-5af9a34e8833, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046405752Z","hostname":"buskin","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:63151","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 9f9b1e33-4da8-44e7-8e80-581afd22135c, rack_id: 3f09cc19-1113-4b10-8b38-5af9a34e8833, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046525564Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046531984Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046572074Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046579364Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046583264Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046587054Z","hostname":"buskin","pid":1590,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046592694Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046598934Z","hostname":"buskin","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046627505Z","hostname":"buskin","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046635625Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.046639685Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.049258465Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.056647588Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.066358019Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.076506584Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.085387985Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.0946637Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.10349468Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.112051827Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.120627154Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.129137041Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.129176351Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.129180862Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.129185062Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.129189382Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.129195462Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.145489667Z","hostname":"buskin","pid":1590,"sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:352"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.145556388Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.145568418Z","hostname":"buskin","pid":1590,"sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:364"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.145637818Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 87.321803ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.148650263Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7423aeaa-1393-484a-aaa7-3f814b441c9e\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:16:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7423aeaa-1393-484a-aaa7-3f814b441c9e\" }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.149459182Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"attempting to create encrypted zvol: name=\"oxi_392736c8-1ed9-4a79-90fc-44745da3ef39/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.157756486Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.167100202Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_392736c8-1ed9-4a79-90fc-44745da3ef39/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.187608365Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_392736c8-1ed9-4a79-90fc-44745da3ef39/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.187655725Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.19249349Z","hostname":"buskin","pid":1590,"sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:379"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.192508Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_392736c8-1ed9-4a79-90fc-44745da3ef39/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.19251855Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.23653206Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 162.229637ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.237045986Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3dc9db42-0db4-4696-b9ab-10b6c59b3f8a\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:16:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3dc9db42-0db4-4696-b9ab-10b6c59b3f8a\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.268833486Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.401012246Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 110.66384ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.401711334Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a3975794-0994-4bb6-a7c4-826a25adce12\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:16:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a3975794-0994-4bb6-a7c4-826a25adce12\" }"} {"msg":"Mounting oxi_392736c8-1ed9-4a79-90fc-44745da3ef39/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.487334816Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.49915925Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.499225571Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.499247171Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.499266231Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.514249301Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 123.22479ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.514646176Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d255989d-c241-4c1d-bddc-7e6cc6eaf73c\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:16:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d255989d-c241-4c1d-bddc-7e6cc6eaf73c\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.638727514Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 237.840528ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.63923681Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2afe0420-afb6-426d-b6ae-69956c894f15\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:16:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2afe0420-afb6-426d-b6ae-69956c894f15\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.878166381Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 282.755576ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:42.87891203Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7723b5bd-a345-4513-aa4c-6e51b73d59c5\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:16:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7723b5bd-a345-4513-aa4c-6e51b73d59c5\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.164866044Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.220487335Z","hostname":"buskin","pid":1590,"sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.221108532Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.221140072Z","hostname":"buskin","pid":1590,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.221214053Z","hostname":"buskin","pid":1590,"file":"oximeter/producer/src/lib.rs:284","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:284","interval":"30s","address":"[fd00:1122:3344:101::1]:49879","producer_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","route":"//9f9b1e33-4da8-44e7-8e80-581afd22135c"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:43.222437147Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"138.058116ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.26675107Z","hostname":"buskin","pid":1590,"component":"VmmReservoirManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.26681162Z","hostname":"buskin","pid":1590,"component":"VmmReservoirManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.266906961Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.266912832Z","hostname":"buskin","pid":1590,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.26768789Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.267720181Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.267750161Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.267765771Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.268040854Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.268273497Z","hostname":"buskin","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.268293967Z","hostname":"buskin","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:63151","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.268346348Z","hostname":"buskin","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.268354768Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.268389528Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.268394789Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.476680652Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.476748683Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.476773693Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.476805534Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.476839004Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:43.477631723Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"409.681547ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:43.481321205Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.509032429Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34677"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.509231682Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.509328612Z","hostname":"buskin","pid":1590,"uri":"/inventory","method":"GET","req_id":"907345b0-d3b6-4344-9398-0c78430edc21","remote_addr":"[fd00:1122:3344:101::1]:34677","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.539784948Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34375"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.539888419Z","hostname":"buskin","pid":1590,"uri":"/sled-role","method":"GET","req_id":"9b279e4d-f92d-4144-9ea7-48cd14b5747d","remote_addr":"[fd00:1122:3344:101::1]:34375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.540561137Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.540577337Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.54087943Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.541079883Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 0b3ac03c-7f08-4aa8-8f85-f7376ecd7697, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool) }, OmicronPhysicalDiskConfig { id: 11993cd8-ddba-452a-84f7-a8bdb9f91f2f, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool) }, OmicronPhysicalDiskConfig { id: 22e8723a-896b-4156-826a-d43be48285d9, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool) }, OmicronPhysicalDiskConfig { id: a33192cc-bbec-4de8-ad3d-aae24f71df2a, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool) }, OmicronPhysicalDiskConfig { id: 50cc043c-fd05-4b72-869e-40cb5840286e, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool) }, OmicronPhysicalDiskConfig { id: a04dd174-c66e-44db-ab33-99986b280cfd, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool) }, OmicronPhysicalDiskConfig { id: e82e14ae-120c-47a6-bd20-10b506220d14, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool) }, OmicronPhysicalDiskConfig { id: f342450b-0679-4d9a-81db-f5bcc40eaa27, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool) }, OmicronPhysicalDiskConfig { id: 940a6c3f-3e38-4791-9d6b-99b9618694af, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.571809901Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:346"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.571934173Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:65163"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: 0b3ac03c-7f08-4aa8-8f85-f7376ecd7697, pool_id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: 11993cd8-ddba-452a-84f7-a8bdb9f91f2f, pool_id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: 22e8723a-896b-4156-826a-d43be48285d9, pool_id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: a33192cc-bbec-4de8-ad3d-aae24f71df2a, pool_id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: 50cc043c-fd05-4b72-869e-40cb5840286e, pool_id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: a04dd174-c66e-44db-ab33-99986b280cfd, pool_id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: e82e14ae-120c-47a6-bd20-10b506220d14, pool_id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: f342450b-0679-4d9a-81db-f5bcc40eaa27, pool_id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: 940a6c3f-3e38-4791-9d6b-99b9618694af, pool_id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.572213126Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.572229986Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.572240966Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.572248337Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.572277577Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.572289327Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.572294947Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.572301177Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.689892721Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:43.689919142Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:43.889637278Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"511.014654ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:44.326580537Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:44.526759218Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.753013997s"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:44.724567753Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:44.904341293Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:44.904419194Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:44.904446114Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:44.904456034Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:44.904464984Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:44.904572465Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:44.904580956Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:45.279820923Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:45.501348277Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:45.503628653Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_eaa88cfe-d08c-4ee9-8c11-845f26835ca2/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:45.717776023Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:45.885717819Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 57.069989ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:46.636540469Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:46.637874934Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 84.062398ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:47.1839464Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:47.809099174Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.753737587s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:47.809185995Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:47.809223456Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 251.756542ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:47.809324817Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 496.29618ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:48.710875007Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_eaa88cfe-d08c-4ee9-8c11-845f26835ca2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:48.727534436Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:48.939962756Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:48.945984385Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:48.945998125Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:48.946020535Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:48.946026425Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:48.946030805Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:49.174973742Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:49.175009413Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:50.389405173Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:50.620157751Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:50.620194301Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:50.620202012Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:50.620226772Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:50.620238282Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:50.620388474Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:50.620398894Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:51.391322432Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:51.391494104Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 575.447601ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:51.391572175Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:51.665494314Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.702928815s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:51.667749149Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ad3dbf69-5e12-43e4-b601-7fce4e3f33b5/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:51.912395585Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:53.175530608Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:53.176515989Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:53.880224684Z","hostname":"buskin","pid":1590,"component":"VmmReservoirManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Looking for unencrypted datasets in oxp_ad3dbf69-5e12-43e4-b601-7fce4e3f33b5","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:53.978962945Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:53.987610353Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:53.987656053Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:53.987677044Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:53.987686464Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:53.987721274Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:53.997516205Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:53.997560396Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"4b494463-e11e-42a2-b533-5933dfea9e20 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.107545954Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.115445114Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.115490264Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.115498314Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.115518604Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.115531795Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.115696017Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.115706637Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.157290358Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4b494463-e11e-42a2-b533-5933dfea9e20/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.166016577Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:54.178408068Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_4b494463-e11e-42a2-b533-5933dfea9e20","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.313835575Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.322160709Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.32220943Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.32221879Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.32222516Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.32223045Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.33194131Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.331971941Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.446402679Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.454915676Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.454948916Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.454955446Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.455043577Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.455055168Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.455187109Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.455197399Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.498921975Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0112b31e-4dc2-42d6-a774-9294ebc7e996/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.507003617Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_0112b31e-4dc2-42d6-a774-9294ebc7e996","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.647811225Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.655822356Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.655868716Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.655887576Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.655893907Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.655898896Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.665971331Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.666005401Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.785219674Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.793239585Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.793273295Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.793281486Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.793350356Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.793363787Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.793529708Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.793556139Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:54.797458263Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.834702896Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2da812e0-f19d-4c88-af4f-92430c5507c5/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.842450004Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_2da812e0-f19d-4c88-af4f-92430c5507c5","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.982245669Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.990022567Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.990072658Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.990086418Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.990093408Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:54.990101718Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.000688318Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.000720269Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.106426528Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.114170456Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.114206767Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.114217837Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.114254307Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.114265377Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.114411509Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.114418439Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.156645428Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_1bbd5d50-8020-48f2-ba68-03b3ab3e35af/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.164787041Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:55.184079269Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_1bbd5d50-8020-48f2-ba68-03b3ab3e35af","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.310740967Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.319042651Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.319118582Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.319127942Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.319135002Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.319140932Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.328876123Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.328915793Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.438971692Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.44674835Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.446782041Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.446789521Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.446805661Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.446815011Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.446958243Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.446975733Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.487928168Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7b354c5b-27b9-467d-9c0f-816eaba8dbe6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.495611375Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.566749802Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"} {"msg":"Looking for unencrypted datasets in oxp_7b354c5b-27b9-467d-9c0f-816eaba8dbe6","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.644293422Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.652271642Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.652312943Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.652318333Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.652324953Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.652329173Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.661942212Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.661973372Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.772624028Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.780782841Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.780815471Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.780823221Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.780834581Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.780843841Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.780973683Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.780983843Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.819992866Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6464a1b3-39ff-4f6d-bb8d-28b483455b0c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.828039837Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_6464a1b3-39ff-4f6d-bb8d-28b483455b0c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.969296859Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.977770115Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.977820546Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.977829186Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.977843556Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.977848876Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.987434545Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:55.987462965Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.097408493Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.105239442Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.105268682Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.105296723Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.105330773Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.105344373Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.105469535Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.105477825Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.147507712Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_22b3b804-39bb-4b5d-9386-43e03fd38f0f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.155356481Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:56.185405362Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_22b3b804-39bb-4b5d-9386-43e03fd38f0f","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.297108879Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.305325972Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.305385543Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.305401993Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.305584395Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 392736c8-1ed9-4a79-90fc-44745da3ef39 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 49112153-160e-48f4-abfa-c3642e83cda8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.305676147Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.30601911Z","hostname":"buskin","pid":1590,"uri":"/omicron-physical-disks","method":"PUT","req_id":"f46de542-259a-40a2-8190-a8e870edf6d9","remote_addr":"[fd00:1122:3344:101::1]:65163","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12734003,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.336640878Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.336772109Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35629"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.336996462Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.337010102Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.419476578Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.419520468Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.479775692Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.479823963Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.540001586Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.540083536Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.540097897Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.540184388Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.587375543Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.587415744Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.587421484Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.642675591Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.646791397Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.650764452Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.664400597Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.668771157Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:56.675460793Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:56.796532917Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:57.187781137Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:57.371482601Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.990633021s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:16:58.189974209Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:58.846343047Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:58.846415078Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:58.846555329Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:59.329735042Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:59.809950011Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:59.810057572Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:59.810178494Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a\") failed: Property not found. retry in 60.906753ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:59.817491686Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:59.817538117Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:16:59.820562251Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a\") failed: Property not found. retry in 80.8465ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:59.888961668Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc\") failed: Property not found. retry in 67.279387ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:59.943890391Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1\") failed: Property not found. retry in 67.770139ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:59.955200619Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a\") failed: Property not found. retry in 246.181811ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:16:59.977473451Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc\") failed: Property not found. retry in 132.030783ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:00.109429398Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1\") failed: Property not found. retry in 122.445961ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:00.137469887Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a\") failed: Property not found. retry in 526.85547ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:00.249078663Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc\") failed: Property not found. retry in 150.952871ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:00.268455033Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1\") failed: Property not found. retry in 108.041021ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:00.291146871Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1\") failed: Property not found. retry in 270.653998ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:00.420889283Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc\") failed: Property not found. retry in 422.577299ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:00.432409144Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1\") failed: Property not found. retry in 407.089543ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:00.700994431Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a\") failed: Property not found. retry in 488.545536ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:00.784788782Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc\") failed: Property not found. retry in 685.484435ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:00.863573616Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1\") failed: Property not found. retry in 910.879299ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:01.117973802Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a\") failed: Property not found. retry in 606.945088ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:01.28345847Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc\") failed: Property not found. retry in 1.48426277s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:01.558763544Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:01.610056426Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:01.610077887Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a\") failed: Property not found. retry in 1.419923114s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:01.900004277Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1\") failed: Property not found. retry in 1.46677629s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:02.03867562Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:feda:9120","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:02.173073786Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:02.178666429Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:02.303542766Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:02.543088314Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:02.732671166Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:03.025493148Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"} {"msg":"wait for service Some(\"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc\") failed: Property not found. retry in 762.310664ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:03.054905472Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:03.217612678Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"} {"msg":"wait for service Some(\"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a\") failed: Property not found. retry in 599.897818ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:03.328904231Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:03.387983052Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"} {"msg":"wait for service Some(\"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1\") failed: Property not found. retry in 1.357551076s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:03.515325617Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:03.591659693Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:03.761064315Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:03.770644944Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:03.770684814Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:03.770694704Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc\") failed: Property not found. retry in 568.945592ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:03.826583049Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a\") failed: Property not found. retry in 1.101615864s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:03.937406956Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.322586056Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.328418232Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"wait for service Some(\"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc\") failed: Property not found. retry in 818.734536ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:04.404522096Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.469940069Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.567114581Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.589106811Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.683483762Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.812499556Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.844952794Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.880247125Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:589"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.880336356Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:627"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.881476559Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:630","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.881485749Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"wait for service Some(\"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1\") failed: Property not found. retry in 904.750381ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:04.884535373Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961476166Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961554067Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961564157Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961570628Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961576957Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961581848Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961588148Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961595458Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961601558Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961607318Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961613458Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961629918Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961636278Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961642388Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961648398Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961654698Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961660478Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961665898Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961669929Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961674099Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961678369Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961682589Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:04.961686549Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:05.809602451Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_76cccebb-a2fc-433e-9266-4ea7f002ac0a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:05.809667681Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_7f76f1e3-d67e-418e-855e-dccd3164c0dc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:05.809675101Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_aa7e09fe-9652-4e85-a32d-c955237752d1"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:05.809687152Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:05.810001205Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:05.810309419Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b7833dcc-a728-445a-a629-1bd1bfa81ffc","remote_addr":"[fd00:1122:3344:101::1]:35629","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9473469,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:05.811984828Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"failed to write DNS configuration (will retry in 133.432335ms)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:06.007238213Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:567","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.164798661Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.216036282Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.275216314Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:583"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.275230134Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.307264538Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.307957245Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.308019596Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.336244926Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.336366788Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50570"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.33660232Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.33660848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.358431578Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.358483369Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.377052259Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.37708808Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.39647347Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.39654026Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.396759043Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.396903415Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.409160004Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.441861275Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.442521912Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.442535462Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.487602634Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 174, 100]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.487648385Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.488382753Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.488392613Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.488673686Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.489755838Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.512956812Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:06.530958026Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:07.632756458Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:07.632876189Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b\") failed: Property not found. retry in 66.474817ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:07.768880783Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b\") failed: Property not found. retry in 80.933154ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:07.848896771Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b\") failed: Property not found. retry in 129.433793ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:07.951966201Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b\") failed: Property not found. retry in 403.577787ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:08.093052721Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:08.30094403Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:08.301135842Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"526b4c29-06fd-4cd3-aa70-b57ee814a98a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48405","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b\") failed: Property not found. retry in 1.125994579s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:08.505989237Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b\") failed: Property not found. retry in 521.031441ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:09.643815448Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b\") failed: Property not found. retry in 959.182855ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:10.174251987Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b\") failed: Property not found. retry in 843.611063ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:11.142927349Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b\") failed: Property not found. retry in 536.552406ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:11.995583074Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b\") failed: Property not found. retry in 1.295909585s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:12.541323597Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:13.857459272Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:13.857568453Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:13.857936177Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:13.858324312Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"6d6f7b6a-321b-46c6-834c-73186cc20d5e","remote_addr":"[fd00:1122:3344:101::1]:50570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7521916,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:13.86080435Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:624"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:13.890865911Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:13.891078543Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58301"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:13.900973306Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:13.901124677Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"7b22d798-b739-4bcb-8db1-11f43613c1cd","remote_addr":"[fd00:1122:3344:101::1]:58301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9893,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:13.903826678Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:14.01516199Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:14.015299372Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42734"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:18.023986499Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:18.024123431Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"57a64dd5-2d36-44ba-9b70-e13de9cbdfe8","remote_addr":"[fd00:1122:3344:101::1]:42734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008679,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:18.026972603Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:18.230590014Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:18.230716535Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33136"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:18.23910949Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:18.239207981Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"2c2b33dc-75c5-49b0-8a80-ba36b25c4f52","remote_addr":"[fd00:1122:3344:101::1]:33136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8333,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:18.24086887Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:18.38101165Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:18.381127602Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54210"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_640a4d7b-43e4-41e4-914c-798fafc7477b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.390669669Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.390923622Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"44616480-30b5-4b9e-888a-ea28c8e39c6a","remote_addr":"[fd00:1122:3344:101::1]:54210","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009605,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:22.393339749Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.627639718Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.62777597Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.637309638Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"b2d31461-1757-4fc5-bafd-a0e6ed038752","remote_addr":"[fd00:1122:3344:101::1]:50344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9402,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714522637.714912, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.638775775Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:22.638780004Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.825779037Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.825901158Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38378"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.835207164Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"703815ee-7c68-4402-92e2-9d5aebee41dd","remote_addr":"[fd00:1122:3344:101::1]:38378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9127,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714522637.714912, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.836898503Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:22.836906123Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.995559203Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:22.995657144Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:40659"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.00406594Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"578d14f4-6c86-41c9-9f3c-7d92de152241","remote_addr":"[fd00:1122:3344:101::1]:40659","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8270,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714522637.714912, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.005560527Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:23.005564837Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.283745354Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.283837444Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63994"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.291732134Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"4664e81f-8319-4a0a-9395-6b6b47a5567a","remote_addr":"[fd00:1122:3344:101::1]:63994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7775,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714522637.714912, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.293068269Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:23.293072629Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:23.399398306Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"374.036397ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.664908269Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.665054121Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42518"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.672851439Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"eaf53938-c5f3-455f-9b66-e86cbb8aa8b6","remote_addr":"[fd00:1122:3344:101::1]:42518","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7762,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714522637.714912, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.676844844Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:23.676851944Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.3420119Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.342102691Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63511"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.345223567Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.364047451Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"c55b0ff2-8292-4d19-92d4-8ba5d0e10d64","remote_addr":"[fd00:1122:3344:101::1]:63511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":21821,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 3423718664, ip_addr: 204.17.205.8, stratum: 3, ref_time: 1714522642.4529228, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.366056693Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.366062423Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1192"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.394449865Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.394570047Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:40600"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.39484573Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.39485199Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.415306802Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.415353643Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.4344796Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.43452314Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.452828908Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.452886639Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.517067347Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.517131388Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.580197853Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.580249254Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.642328428Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.642379409Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.70149841Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.7015504Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.767333697Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.767423528Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.767665091Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.767798202Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:23.800694375Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:23.800743456Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:23.800750116Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:23.800756406Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:23.800762576Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.800871297Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"85c739db-2c2c-490d-997e-417d1d58d83a","remote_addr":"[fd00:1122:3344:101::1]:40600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\", TimeNotSynchronized), (\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\", TimeNotSynchronized), (\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\", TimeNotSynchronized), (\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\", TimeNotSynchronized), (\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\", TimeNotSynchronized)]","latency_us":406226,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:23.801059699Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"73.408769ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"85c739db-2c2c-490d-997e-417d1d58d83a\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"85c739db-2c2c-490d-997e-417d1d58d83a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.876190972Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.876593707Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.876603947Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.8953885Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.89543437Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.91300859Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.91305459Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.930425738Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.930466668Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.947932456Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.947990297Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.96583908Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.96587843Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.983555361Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:23.983598771Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.003396806Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.003438976Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.02145129Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.02150064Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.021723533Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.021855044Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.055157972Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.055204203Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.055212373Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.055218813Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.055224993Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.055367805Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"ac5181bc-3372-487e-90f3-65c6ad18e80d","remote_addr":"[fd00:1122:3344:101::1]:40600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\", TimeNotSynchronized), (\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\", TimeNotSynchronized), (\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\", TimeNotSynchronized), (\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\", TimeNotSynchronized), (\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\", TimeNotSynchronized)]","latency_us":178993,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.055556197Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"130.352366ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ac5181bc-3372-487e-90f3-65c6ad18e80d\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ac5181bc-3372-487e-90f3-65c6ad18e80d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.186947598Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.187356773Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.187367053Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.207521531Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.207560732Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.226196963Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.226235724Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.244483531Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.244523701Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.262870159Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.26290825Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.281048846Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.281090276Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.299189752Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.299226622Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.317319437Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.317357288Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.335524854Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.335568655Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.335775747Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.335889608Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.36862865Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.36867461Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.36868438Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.36869216Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.368701771Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.368819342Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"d4d2ea11-7a02-4055-8fd1-71574bd8f158","remote_addr":"[fd00:1122:3344:101::1]:40600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\", TimeNotSynchronized), (\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\", TimeNotSynchronized), (\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\", TimeNotSynchronized), (\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\", TimeNotSynchronized), (\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\", TimeNotSynchronized)]","latency_us":181676,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.369012234Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"184.665493ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d4d2ea11-7a02-4055-8fd1-71574bd8f158\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d4d2ea11-7a02-4055-8fd1-71574bd8f158\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.554401218Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.55543227Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.5554623Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.574507296Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.574563856Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.592932365Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.592980885Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.610931039Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.61097802Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.628799942Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.628851192Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.647751977Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.647788678Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.66651675Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.66656492Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.684796167Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.684851928Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.702972854Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.703028594Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.703242597Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.703372378Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.736002968Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.736044939Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.736067819Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.736076019Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.736082449Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.736183161Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"33eb8812-2c32-4e3d-bc6c-db986b32c1fa","remote_addr":"[fd00:1122:3344:101::1]:40600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\", TimeNotSynchronized), (\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\", TimeNotSynchronized), (\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\", TimeNotSynchronized), (\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\", TimeNotSynchronized), (\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\", TimeNotSynchronized)]","latency_us":181594,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:24.736374053Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"86.972329ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"33eb8812-2c32-4e3d-bc6c-db986b32c1fa\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"33eb8812-2c32-4e3d-bc6c-db986b32c1fa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.824510453Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.824834796Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.824845357Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.844425139Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.84448361Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.862516154Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.862565645Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.88062576Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.88067077Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.899499544Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.899528764Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.918626881Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.918671591Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.938603558Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.938644918Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.957783065Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.957820926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.976249295Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.976294785Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.976465497Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:24.976594459Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:25.008746763Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:25.008784244Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:25.008792994Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:25.008801464Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:25.008808094Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.008913345Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"8609fb8a-e3a3-49c7-8e33-42976e1a48d7","remote_addr":"[fd00:1122:3344:101::1]:40600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\", TimeNotSynchronized), (\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\", TimeNotSynchronized), (\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\", TimeNotSynchronized), (\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\", TimeNotSynchronized), (\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\", TimeNotSynchronized)]","latency_us":184246,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:25.009090447Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"139.287405ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8609fb8a-e3a3-49c7-8e33-42976e1a48d7\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8609fb8a-e3a3-49c7-8e33-42976e1a48d7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.150191359Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.150445942Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.150460722Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.169901812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.169951103Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.187605023Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.187653694Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.205543157Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.205603817Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.223578361Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.223622962Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.241782198Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.241833539Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.260088276Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.260135706Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.278314193Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.278356023Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.296643441Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.296679541Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.296882483Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.297009545Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.381127869Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.38119036Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.38119731Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.38120183Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.38120746Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.473817571Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.477722535Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.481787531Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.486213432Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.490994926Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.495687549Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.500389303Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.505141007Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.509929501Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:25.514571043Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:28.743789888Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:28.743971146Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:28.856958996Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:28.857120801Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 51.019391ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:28.898076948Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 58.048521ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:28.959686654Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 170.001635ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:29.05430713Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 62.101186ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:29.090530975Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 56.151381ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:29.203709999Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:29.259850867Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:29.260003325Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:29.260026519Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 406.527928ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:29.362750535Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 287.029841ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:29.379690547Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:29.379747117Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:29.397021892Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:29.397844271Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\") failed: Property not found. retry in 51.858778ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:29.669200211Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 417.664572ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:29.821764727Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\") failed: Property not found. retry in 69.900858ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:29.882240673Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 1.012832883s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:29.893542564Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\") failed: Property not found. retry in 51.270627ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:30.061533426Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\") failed: Property not found. retry in 248.406318ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:30.343772072Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.076300669s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:30.457082943Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\") failed: Property not found. retry in 118.100909ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:30.531025197Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 51.64962ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:30.625027187Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\") failed: Property not found. retry in 367.265179ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:30.667065576Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\") failed: Property not found. retry in 233.668875ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:30.724955814Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 79.763253ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:30.773489227Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 233.223237ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:30.912002839Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 1.0752759s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:30.996533203Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\") failed: Property not found. retry in 379.292857ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:31.106147134Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\") failed: Property not found. retry in 1.030983144s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:31.203271321Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 353.889401ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:31.221422455Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\") failed: Property not found. retry in 409.565466ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:31.497672545Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.177678006s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:31.54833804Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 983.141223ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:31.587170247Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\") failed: Property not found. retry in 1.325908445s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:31.918741141Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 939.63393ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:32.08309352Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\") failed: Property not found. retry in 1.494645381s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:32.247521423Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 1.008240109s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:32.583243212Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 506.437768ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:32.739041669Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 777.38077ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:33.037345828Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.01161109s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:33.263351217Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\") failed: Property not found. retry in 735.291937ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:33.275896904Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 1.12547025s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:33.607825609Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\") failed: Property not found. retry in 573.829185ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:33.756006587Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 1.006088087s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:33.826190467Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\") failed: Property not found. retry in 1.242592458s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:34.023074007Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.187768784s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:34.286929495Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\") failed: Property not found. retry in 585.838146ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:34.339727103Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 972.624954ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:34.744413449Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 1.298639554s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:34.843486192Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\") failed: Property not found. retry in 1.175573653s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:34.934764182Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de\") failed: Property not found. retry in 1.44193788s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:35.277146328Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.209427354s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:35.487738408Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 1.494059497s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:35.728549685Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342\") failed: Property not found. retry in 641.867662ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:36.12368973Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94\") failed: Property not found. retry in 572.375818ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:36.155713952Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.045366019s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:36.711159213Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 516.037711ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:37.236929092Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:37.402793924Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54942"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:37.402905576Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"db52f1b6-1af4-43bd-9c98-5cbe6a7261b2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54942","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":55,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 957.476069ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:37.763998893Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.439139337s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:37.772948375Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:37.878075788Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"280.712322ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 1.121831288s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:38.732534253Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.419543334s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:39.225453296Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 1.206298138s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:40.11847539Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.097761012s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:40.658090773Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 1.373535005s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:41.337391201Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 562.027838ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:41.768538983Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 854.20541ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:42.50054272Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 1.081512576s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:42.722691151Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.041709492s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:43.364834407Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 1.469370015s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:43.813369177Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 1.069843127s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:44.415923274Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 898.948249ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:45.29242594Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 906.96445ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:45.495415673Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3\") failed: Property not found. retry in 1.337027942s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:46.200840368Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f\") failed: Property not found. retry in 996.036139ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:46.410914641Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.55909553Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_84bb5ac4-5b69-491a-915b-56f3458c45f3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.559189911Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_1cf97cd5-2bfc-4f9b-9562-4d2722532342"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.559201461Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_bb0a7b7e-c94d-46a1-b575-2f0b132ffd94"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.559213431Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_2c74ddf5-baea-4359-aeb5-d5897afea56f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.559223331Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_06252ed0-79b6-4cf3-a662-7733f01c14de"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.559260262Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.560164002Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.56090276Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"4bc73c5e-fb0d-4100-8446-6e7709231ae9","remote_addr":"[fd00:1122:3344:101::1]:40600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":22380881,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.5934511Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54936"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.622903074Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:47.86313238Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.493475262Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.493612404Z","hostname":"buskin","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"96e525e7-640d-4557-8ea3-8659960ab56a","remote_addr":"[fd00:1122:3344:101::1]:54936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4900084,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.530837856Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.532302153Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:56214"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.538855487Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.538876507Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.562613607Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.562659607Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.587210166Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.587259026Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.609925554Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.609990484Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.630587798Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.630632849Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.649590734Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.649631564Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.673022719Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.67307045Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.697231364Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.697276095Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.722711603Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.722769284Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.80786462Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.80792013Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.880916839Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.880971999Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.96477322Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:52.964833961Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.044498654Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.044555475Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.11371972Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.11377389Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.184959858Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.185018159Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.198954337Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"1.258196323s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.263789053Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.263840003Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.337990785Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.338041355Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.403805071Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.403855692Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.469045942Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.469097032Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.529435527Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.529482038Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.594311383Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.594988001Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.595834011Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.596286776Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708011554Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708076934Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708083694Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708088304Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708093575Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708097895Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708102155Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708106355Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708109565Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708113635Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708117835Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708121905Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708125185Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708129425Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708133615Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708138525Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708147135Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708151395Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.708156385Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.708835833Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c659066b-ab25-445e-94e3-78662e7bd8c3","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized)]","latency_us":1175940,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:53.709640342Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"60.030997ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c659066b-ab25-445e-94e3-78662e7bd8c3\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c659066b-ab25-445e-94e3-78662e7bd8c3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.7702647Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.770635004Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.770645244Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.793079939Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.793123729Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.814409971Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.814475382Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.83374604Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.833785981Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.854925331Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.854971161Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.876047381Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.876088341Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.896053708Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.896097968Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.918029197Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.918089938Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.938716332Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.938761332Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.959859281Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.959901542Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.984283679Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:53.984323069Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.005911194Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.005956635Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.026845292Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.026898772Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.046947499Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.046993169Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.067894087Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.067941987Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.087497759Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.0875433Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.108436667Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.108494157Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.128602206Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.128648726Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.147401059Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.147445219Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.166782039Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.166821349Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.185673413Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.185719744Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.185964597Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.186117098Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287368697Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287420048Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287426688Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287431268Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287435738Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287440168Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287444718Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287447898Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287453088Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287457308Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287460788Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287465868Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287470068Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287474208Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287478509Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287483718Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287504519Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287508739Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287515839Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.28761141Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"cb260b46-bbd4-4646-ad9a-be05e4e6941e","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized)]","latency_us":517186,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.287786852Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"120.29654ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cb260b46-bbd4-4646-ad9a-be05e4e6941e\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cb260b46-bbd4-4646-ad9a-be05e4e6941e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.409952188Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.410288682Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.410295532Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.433840949Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.43387791Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.453740345Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.453786606Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.473506929Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.47355098Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.49469039Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.4947364Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.514270612Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.514300802Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.533983416Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.534015966Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.553977043Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.554011723Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.571511702Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.571543812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.588488074Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.588519595Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.608762794Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.608798815Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.630059326Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.630093196Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.649421326Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.649451496Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.668849986Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.668883706Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.688293337Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.688328337Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.708632958Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.708664948Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.730668028Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.730706828Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.750712485Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.750748786Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.770840574Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.770869884Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.791159804Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.791193874Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.811482265Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.811526325Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.811774758Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.81193124Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914547524Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914601585Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914608875Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914616115Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914621635Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914626005Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914631595Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914636805Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914642345Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914648455Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914654386Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914660086Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914667216Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914672886Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914680326Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914696326Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914702566Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914708366Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.914714056Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:54.914834537Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"d8ae42c4-c7b4-4049-a26b-bbf1c4085f40","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized)]","latency_us":504741,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:54.91504258Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"130.901294ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d8ae42c4-c7b4-4049-a26b-bbf1c4085f40\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d8ae42c4-c7b4-4049-a26b-bbf1c4085f40\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.04732824Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.048114829Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.048126499Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.070466123Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.070517394Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.08958604Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.089640351Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.111223275Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.111270996Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.130085699Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.13013304Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.150530851Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.150575292Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.169075842Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.169118352Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.188037217Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.188080637Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.210262859Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.21032634Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.234817558Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.234861698Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.254077686Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.254123937Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.273639459Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.273696089Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.290609421Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.290652821Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.308852038Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.308894888Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.32846037Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.328507681Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.345878398Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.345920939Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.362990282Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.363044083Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.379811503Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.379854104Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.400196475Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.400237715Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.417720183Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.417763114Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.436077852Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.436128742Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.436326374Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.436465896Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.53106248Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.53111163Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.53112003Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.53112611Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531133201Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.53113945Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531145021Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531150521Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531157961Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531164061Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531171351Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531177491Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531181791Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531188741Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531210401Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531216161Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531221651Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531227351Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531234502Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.531389113Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"408ad9d7-108a-4d87-9579-935a2bee95f5","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized)]","latency_us":483922,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:55.531606056Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"98.59247ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"408ad9d7-108a-4d87-9579-935a2bee95f5\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"408ad9d7-108a-4d87-9579-935a2bee95f5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.631814853Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.632804914Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.632819594Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.659445746Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.659495317Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.680693748Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.680742538Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.701684926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.701747286Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.724742178Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.724798258Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.747913461Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.747965561Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.771439257Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.771506018Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.793453327Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.793507268Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.815098743Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.815151584Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.83865047Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.838706081Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.862362739Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.8624138Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.88531971Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.885385961Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.909327842Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.909376023Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.933884101Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.933939741Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.956076223Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.956124073Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.978848861Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:55.978897111Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.002193716Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.002250767Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.022138242Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.022188363Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.043532474Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.043581915Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.066696627Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.066753038Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.089587887Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.089655518Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.089894081Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.090053352Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201429996Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201478207Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201484917Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201489727Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201494247Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201498507Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201503967Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201508217Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201512747Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201516867Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201521337Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201525597Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201529757Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201534957Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201547417Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201553297Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201557738Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201563238Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201567708Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.201710509Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"64b21d0f-fd5a-4606-9d5e-25758137ae3d","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized)]","latency_us":569731,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.201906652Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"108.15766ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"64b21d0f-fd5a-4606-9d5e-25758137ae3d\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"64b21d0f-fd5a-4606-9d5e-25758137ae3d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.311380094Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.311734258Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.311741398Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.336468979Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.336509899Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.357634789Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.357672689Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.378781749Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.378820429Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.400117011Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.400156911Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.420764615Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.420801965Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.441783874Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.441832674Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.463336318Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.463370959Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.484855992Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.484891523Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.5084362Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.508474631Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.530100786Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.530141926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.552114576Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.552158486Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.574417979Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.574456769Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.597785524Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.597824714Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.62121962Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.62125802Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.643744495Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.643784416Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.665785926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.665823826Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.687753275Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.687794525Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.710274961Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.710312321Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.732954338Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.732994698Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.754477392Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.754520043Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.754755235Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.754903547Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.856441119Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.85649388Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.85650198Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.85650759Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.85651317Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.85651877Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.8565242Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.85653122Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.85653677Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.85654069Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.85654616Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.85655149Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.856557041Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.856579611Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.856585521Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.856590911Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.856598181Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.856605211Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.856609061Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:56.856755613Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"4ba30807-9087-4068-9703-b0a38081175e","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized)]","latency_us":545212,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:56.856932695Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"215.139166ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4ba30807-9087-4068-9703-b0a38081175e\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4ba30807-9087-4068-9703-b0a38081175e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.073166388Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.073555233Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.073573713Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.101989615Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.102064746Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.127159601Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.127209592Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.152056444Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.152113994Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.179296683Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.179345633Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.205323488Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.205371159Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.231976271Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.232047871Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.257938805Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.257993336Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.285398607Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.285453257Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.31298877Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.31304513Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.338847033Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.338899594Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.363294231Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.363344491Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.387368734Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.387415674Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.41519831Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.41525199Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.440325885Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.440379946Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.465698283Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.465758674Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.490235911Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.490297542Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.516411088Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.516463489Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.541575604Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.541627915Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.5659183Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.565973121Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.591255878Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.591333199Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.591548811Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.591694143Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715213614Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715276905Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715286705Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715295655Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715306045Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715314715Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715324395Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715332496Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715341186Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715351106Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715360446Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715367936Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715377836Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715398007Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715406767Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715415327Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715418827Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715422357Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715426047Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:57.715547308Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b5637f84-4564-4af8-8e03-f1a0101a2a47","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized)]","latency_us":642224,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:57.715771651Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"357.830655ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b5637f84-4564-4af8-8e03-f1a0101a2a47\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b5637f84-4564-4af8-8e03-f1a0101a2a47\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.075044907Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.075417881Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.075426051Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.099488984Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.099539565Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.121647156Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.121687806Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.142775455Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.142817866Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.16518745Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.16523028Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.188235241Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.188271772Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.209680765Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.209715385Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.231249969Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.2312877Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.254061718Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.254100619Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.275842575Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.275878866Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.296928055Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.296964405Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.317268665Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.317303256Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.339536668Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.339572799Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.360309994Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.360346454Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.381507205Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.381549035Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.405401726Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.405450536Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.425298111Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.425330832Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.445092006Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.445133156Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.464162332Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.464192693Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.484143049Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.48417697Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.503753272Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.503794702Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.504011495Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.504140086Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605206743Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605256204Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605265164Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605269614Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605273844Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605278014Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605283544Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605289294Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605294654Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605300414Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605305854Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605311514Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605335974Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605341714Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605346805Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605353855Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605359065Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605364245Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605370835Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.605487356Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"13348f3d-2c38-4e5a-ac16-2684f5bb6ee0","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized)]","latency_us":530256,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:58.605720779Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"262.712447ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"13348f3d-2c38-4e5a-ac16-2684f5bb6ee0\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"13348f3d-2c38-4e5a-ac16-2684f5bb6ee0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.869976588Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.870319272Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.870326612Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.894452375Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.894516566Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.916484575Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.916546306Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.93888995Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.93894119Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.961379425Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.961428915Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.983720658Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:58.983770169Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.004234591Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.004283582Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.02444007Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.024494901Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.046705642Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.046752873Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.068290687Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.068335458Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.089373517Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.089430697Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.110944421Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.110985842Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.133618329Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.13368778Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.156465258Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.156516699Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.177973792Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.178032333Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.200466557Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.200525238Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.223051014Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.223105144Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.244304575Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.244356325Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.26595533Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.266003541Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.289904222Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.289960033Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.312362937Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.312424408Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.31264162Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.312777092Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415043032Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415112173Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415122123Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415128713Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415135253Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415140443Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415146883Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415154923Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415163324Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415171234Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415179164Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415197734Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415206714Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415215034Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415221404Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415229604Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415237504Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415245664Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415252095Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.415343766Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"511fe5b1-4791-455f-8f92-32c8dbf545be","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized)]","latency_us":545215,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:17:59.415553658Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"406.9246ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"511fe5b1-4791-455f-8f92-32c8dbf545be\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"511fe5b1-4791-455f-8f92-32c8dbf545be\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.823060642Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.823389906Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.823400276Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.84749262Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.84754751Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.86950865Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.86955575Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.88891261Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.88896369Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.911375905Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.911422085Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.93212021Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.932178521Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.952114597Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.952158927Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.974147117Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.974203378Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.996012635Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:17:59.996064135Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.021585035Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.021631606Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.043850858Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.043906749Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.066855529Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.06690353Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.089453095Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.089501366Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.110470504Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.110514724Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.132100039Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.13214831Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.152354839Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.15240073Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.17441323Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.1744624Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.195181915Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.195236656Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.222254252Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.222301683Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.245879281Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.245927791Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.267677668Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.267735389Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.267936721Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.268104283Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367651382Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367715813Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367725253Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367730943Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367736513Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367742203Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367748113Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367755144Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367760713Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367767713Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367772734Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367798744Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367806424Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367811664Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367818654Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367824314Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367829834Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367836994Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.367842394Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.367936626Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b0c304c8-1a0e-4bd5-8cea-69e9337af772","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized)]","latency_us":544730,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:00.368156578Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"498.040056ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b0c304c8-1a0e-4bd5-8cea-69e9337af772\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:17:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b0c304c8-1a0e-4bd5-8cea-69e9337af772\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.867486764Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.867824478Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.867831568Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.889867758Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.889919359Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.90762533Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.90767558Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.927728178Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.927779159Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.946986806Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.947037347Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.965222313Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.965279414Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.984949297Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:00.984999758Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.005229947Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.005280598Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.027133676Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.027183167Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.048766411Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.048809411Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.07251749Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.072586481Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.096527693Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.096574113Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.118087018Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.118134148Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.140548292Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.140596203Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.162596943Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.162637973Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.182758121Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.182795882Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.200124168Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.200162899Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.220446869Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.22049927Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.241847312Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.241886332Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.262292914Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.262345084Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.282430172Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.282481223Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.282680205Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.282854877Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381105052Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381167483Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381175383Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381181023Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381186313Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381191593Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381197513Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381203363Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381209053Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381214723Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381231244Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381238384Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381243544Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381250284Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381255734Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381262884Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381268334Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381273934Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381279294Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:01.381395166Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1641092e-3e9b-4a10-9aff-b078007eef59","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized)]","latency_us":513767,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:01.381607338Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"842.913259ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1641092e-3e9b-4a10-9aff-b078007eef59\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1641092e-3e9b-4a10-9aff-b078007eef59\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.225199731Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.226314783Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.226325883Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.249600288Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.249648518Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.271070041Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.271118062Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.292021369Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.29207326Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.311799124Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.311846674Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.332215835Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.332276976Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.351943449Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.3519822Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.372824136Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.372866946Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.392398638Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.392440468Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.412214913Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.412257193Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.430882745Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.430921015Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.452353279Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.452396589Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.472457487Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.472499387Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.492968169Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.49301335Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.514518764Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.514567925Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.535525162Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.535569473Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.555081344Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.555132675Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.575075971Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.575117712Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.595360141Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.595404022Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.616186208Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.616232118Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.635999192Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.636048333Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.636262446Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.636398707Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740161025Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740215865Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740222275Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740229435Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740234905Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740240115Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740245395Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740250306Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740256975Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740262276Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740282266Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740286356Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740290146Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740293766Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740298786Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740302756Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740307746Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740312626Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740317546Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:02.740377707Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e89b4285-60ed-48bc-a2cf-7ffa7c2ae940","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized)]","latency_us":515010,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:02.740563579Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"381.797123ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e89b4285-60ed-48bc-a2cf-7ffa7c2ae940\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e89b4285-60ed-48bc-a2cf-7ffa7c2ae940\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.123634075Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.123974129Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.123993Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.147705258Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.147752019Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.171212525Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.171255236Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.193797682Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.193843442Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.214121132Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.214160713Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.235010919Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.23506131Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.257030829Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.25707073Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.279070979Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.27911839Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.298801323Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.298840954Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.319999884Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.320041804Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.341137964Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.341179194Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.361195831Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.361238942Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.382298991Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.382339111Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.403704544Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.403746394Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.424215506Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.424262897Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.445906922Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.445946493Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.467519468Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.467560928Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.486863507Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.486910488Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.506888314Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.506928355Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.527149935Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.527201205Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.548384825Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.548434236Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.548623648Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.5487461Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650674996Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650747287Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650755617Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650762857Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650768647Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650774347Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650779977Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650786827Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650792627Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650806388Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650810658Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650814668Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650818538Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650824068Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650831518Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650837008Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650842208Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650845808Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.650849548Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:03.650894139Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b971a2e8-ce85-4c40-87f6-e4171e67839b","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized)]","latency_us":527115,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:03.651086611Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.086312897s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b971a2e8-ce85-4c40-87f6-e4171e67839b\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b971a2e8-ce85-4c40-87f6-e4171e67839b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.73856315Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.739004835Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.739028646Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.762132508Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.762170278Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.780937861Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.780974612Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.801354463Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.801391163Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.820908335Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.820961725Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.839106671Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.839143552Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.858955017Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.858998917Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.879221407Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.879263197Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.899157303Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.899199183Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.919738886Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.919786017Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.940124788Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.940167868Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.960293347Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.960328467Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.980710888Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.980746069Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.999775865Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:04.999810565Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.019438518Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.019485738Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.039759258Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.039792218Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.058877875Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.058916535Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.07878217Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.078820151Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.098491004Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.098523745Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.118157887Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.118191918Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.137730669Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.13776689Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.137980062Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.138098614Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243666921Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243724882Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243729992Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243734062Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243738122Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243744082Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243749342Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243754982Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243774453Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243780243Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243784133Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243789083Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243792643Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243796203Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243799873Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243803303Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243806873Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243810583Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243814433Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:05.243818093Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"cd3cb844-bd6c-408e-829f-2e0e55bb8e29","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized)]","latency_us":505033,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:05.243962535Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.188954498s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cd3cb844-bd6c-408e-829f-2e0e55bb8e29\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cd3cb844-bd6c-408e-829f-2e0e55bb8e29\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.434450574Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.434801388Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.434828289Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.458114403Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.458155213Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.478007979Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.478048079Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.497924305Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.497966595Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.516572016Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.516604357Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.535933976Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.535980886Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.555227865Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.555261405Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.574093099Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.574128329Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.592417437Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.592457607Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.61118227Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.6112248Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.629363126Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.629401726Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.647258189Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.6472977Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.665669878Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.665708498Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.684160708Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.684192518Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.702340814Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.702377715Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.720779863Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.720809754Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.739981612Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.740010172Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.759124809Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.759176039Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.77777754Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.777805821Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.796188459Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.79622968Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.814321375Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.814371706Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.814585488Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.81473392Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.90551422Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.90555249Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.90556089Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.90556654Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.90557184Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.90557694Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905582231Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905588841Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905603541Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905607741Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905611331Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905616421Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905619981Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905623591Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905627271Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905630861Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905636171Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905641191Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905646151Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:06.905704822Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b227cee0-36f9-4417-a7b5-672e48efca5b","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized)]","latency_us":471122,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:06.905881964Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.627367577s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b227cee0-36f9-4417-a7b5-672e48efca5b\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b227cee0-36f9-4417-a7b5-672e48efca5b\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:07.437402765Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:07.440041555Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"a0c5b8c7-1d81-4924-bc12-1ec5efb0120c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47533","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1128,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.534689097Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.535621157Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.535628848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.557822749Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.55787573Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.577565013Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.577611884Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.597354778Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.597403729Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.615966109Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.61600992Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.633704541Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.633751751Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.651611994Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.651665954Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.671244636Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.671291187Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.689466633Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.689520384Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.708212556Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.708264696Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.728603377Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.728645108Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.74819503Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.74823871Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.766667059Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.76671093Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.784825085Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.784865626Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.802715678Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.802760479Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.820662822Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.820701692Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.838866469Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.838925729Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.856291826Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.856341927Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.874617254Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.874676785Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.892872082Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.892913402Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.911962268Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.912009669Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.912226331Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:08.912345763Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.00292624Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.002982871Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.002991621Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.002998161Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003024081Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003030841Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003038761Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003045152Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003052862Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003059272Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003065522Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003071812Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003076122Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003081982Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003086242Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003090482Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003094892Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003099262Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003104952Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:09.003181083Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"3fc6351f-753b-42e4-a806-695d22e974cc","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized)]","latency_us":468328,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.003387236Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.450374734s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3fc6351f-753b-42e4-a806-695d22e974cc\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3fc6351f-753b-42e4-a806-695d22e974cc\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:09.513099429Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"2.598399426s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.454420011Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.455372482Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.455380842Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.478513044Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.478565375Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.497914075Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.497965935Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.515849898Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.515893208Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.532473377Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.532512187Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.55128356Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.551321821Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.569030082Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.569070112Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.586299598Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.586346308Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.604664416Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.604710076Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.621976652Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.622021913Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.638992636Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.639035826Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.656367593Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.656414093Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.674302506Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.674355077Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.6913995Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.69143846Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.71074481Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.71078296Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.728485191Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.728523181Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.744891947Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.744931288Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.76271754Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.76276399Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.780011126Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.780050406Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.801632431Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.801672451Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.820478765Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.820525865Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.820754808Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.82090761Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916726287Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916789288Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916795908Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916803028Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916808478Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916813698Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916820238Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916825218Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916832828Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916838378Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916843749Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916849389Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916856009Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916861409Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916867169Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916870919Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916886319Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916890399Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.916894219Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:10.91695661Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5c727c05-06b4-4e25-bc8f-69b3c331d1bf","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized)]","latency_us":462407,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:10.917145772Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.632848069s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5c727c05-06b4-4e25-bc8f-69b3c331d1bf\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5c727c05-06b4-4e25-bc8f-69b3c331d1bf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.551575296Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.552219523Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.552245303Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.574748009Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.574804349Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.596372434Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.596444615Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.614402088Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.61447378Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.633542336Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.633602397Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.651202206Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.651249917Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.669833678Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.669885278Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.687888762Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.687941923Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.706926838Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.706971589Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.724250675Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.724294626Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.742904277Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.742963278Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.761129994Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.761175264Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.780029858Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.780089909Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.800510251Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.800551961Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.819515276Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.819562337Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.839481503Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.839532893Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.857953132Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.858013923Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.876057198Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.876124518Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.893726918Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.893783959Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.911889584Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.911945865Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.93000699Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.930078071Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.930293843Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:13.930449695Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.022415359Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.022482459Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.02250143Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.02251455Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.02252441Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.02253529Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.02254106Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.02254858Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.02255419Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.02256013Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.02256766Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.022575651Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.022585261Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.022591271Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.022598891Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.022606291Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.022623611Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.022630081Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.022635501Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:14.022643171Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"ae3a0289-d22c-4d9e-bb07-474a7adc4e37","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized)]","latency_us":470783,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:14.023258288Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.785642076s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ae3a0289-d22c-4d9e-bb07-474a7adc4e37\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ae3a0289-d22c-4d9e-bb07-474a7adc4e37\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.810186352Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.810642028Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.810655978Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.83289981Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.832960821Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.853119559Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.85317201Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.871851512Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.871897483Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.889697235Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.889749525Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.909121915Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.909169075Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.927098909Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.92714912Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.944909591Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.944952552Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.961211366Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.961252267Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.979020558Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.979065279Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.995980811Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:16.996025931Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.012540239Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.012586289Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.029267398Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.029304879Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.046964299Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.04699975Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.067683294Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.067727785Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.085069982Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.085113212Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.104252059Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.10430691Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.121600356Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.121641437Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.141217659Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.141269669Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.160045573Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.162375989Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.178527372Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.178593823Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.178792855Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.178915387Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272182665Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272255336Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272266236Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272276226Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272286266Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272295946Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272306236Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272314227Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272324087Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272334157Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272342417Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272351877Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272370657Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272381107Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272391247Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272429708Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272436318Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272442348Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272450168Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:17.272528319Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"ea1e5638-0cc8-455d-bdcf-eceb230e27f2","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized)]","latency_us":462118,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:17.272741071Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.928228143s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ea1e5638-0cc8-455d-bdcf-eceb230e27f2\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ea1e5638-0cc8-455d-bdcf-eceb230e27f2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.202906954Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.203287388Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.203301728Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.226749934Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.226805385Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.245606408Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.247988666Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.264474442Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.264524603Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.282311785Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.282367376Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.300949967Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.301005427Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.319678059Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.31973893Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.340508675Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.340558016Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.360864596Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.360928327Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.380504109Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.38057077Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.39992897Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.39997787Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.418856464Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.421245401Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.43696691Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.437019791Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.454865023Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.454910343Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.473825898Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.473880259Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.494625184Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.494674565Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.512558478Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.512630629Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.531884957Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.531947718Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.549897272Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.549943702Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.568143679Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.568201599Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.58682862Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.586881241Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.587142274Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.587291686Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679003877Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679053757Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679063607Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679069757Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679076927Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679085398Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679092518Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679100908Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679108128Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679122728Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679129948Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679139398Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679147828Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679154388Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679161258Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679171239Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679178259Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679183068Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679188659Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:19.6792773Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"82a83914-d759-4094-824e-0725d1bfd913","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized)]","latency_us":476213,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:19.679525552Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.019194619s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"82a83914-d759-4094-824e-0725d1bfd913\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"82a83914-d759-4094-824e-0725d1bfd913\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.699201367Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.69953934Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.699549291Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.722922016Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.722963456Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.743110955Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.743150135Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.763523077Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.763576657Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.783112019Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.78315848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.801034812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.801083533Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.819083357Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.819141798Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.838020372Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.838066403Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.854989924Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.855027805Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.873468534Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.873526035Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.892216197Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.892260907Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.913066424Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.913134504Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.934277574Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.934350475Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.953945707Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.953994628Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.973937594Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.973983015Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.993220233Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:22.993272814Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.012857126Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.012903516Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.031917072Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.031963173Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.050725365Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.050776836Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.06882845Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.068880541Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.087236659Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.08729595Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.087522413Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.087651884Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184008378Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184052168Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184058088Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184064998Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184071348Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184079148Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184085018Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184090809Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184096979Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184102709Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184108279Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184115479Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184121009Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184127829Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184146409Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184151339Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184159889Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184163959Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184169859Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:23.18423895Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a71be824-bd13-4a54-9194-fabf04fab967","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized)]","latency_us":484910,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:23.184436933Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.914215398s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a71be824-bd13-4a54-9194-fabf04fab967\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a71be824-bd13-4a54-9194-fabf04fab967\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.099513178Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.099869822Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.099881082Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.122917953Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.122990714Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.141578165Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.141631876Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.148457843Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"3.496914385s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.158500977Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.158545437Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.178884408Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.178934599Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.196579839Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.1966404Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.213170757Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.213222228Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.230888268Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.230927819Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.248403557Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.248453318Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.267343012Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.267386923Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.285144064Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.285203275Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.303860546Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.303920167Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.320815159Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.32086589Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.339052186Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.339095066Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.357762448Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.357818219Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.375791793Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.375850093Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.393935969Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.393974889Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.411971583Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.412032224Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.43013903Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.43018133Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.44869965Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.448743061Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.465611522Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.465665643Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.465883945Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.466010477Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556654335Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556717836Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556728256Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556734976Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556742186Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556750786Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556758246Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556766216Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556772967Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556786217Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556794677Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556812377Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556819417Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556824097Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556830617Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556835327Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556840207Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556845027Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.556852657Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:27.556917148Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"95dc3556-0066-4438-9cfd-a164a76f63f5","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized)]","latency_us":457239,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:27.557127031Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.301947257s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"95dc3556-0066-4438-9cfd-a164a76f63f5\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"95dc3556-0066-4438-9cfd-a164a76f63f5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.86083208Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.861202604Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.861211694Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.883558468Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.883611778Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.903764937Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.903819038Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.921583729Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.92164144Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.941202462Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.943503058Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.959086195Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.959140236Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.977933979Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.977990879Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.995581539Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:33.995636239Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.013395501Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.013445332Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.031361955Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.031416065Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.048954314Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.049009304Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.06977943Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.069828341Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.087921226Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.087973157Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.105538076Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.105585146Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.126379993Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.126427763Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.146241258Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.148641935Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.164716188Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.164767848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.183533601Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.183597702Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.204062524Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.204117205Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.222438773Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.222495763Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.241410038Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.241464868Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.241719091Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.241873483Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335232132Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335314304Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335327644Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335336364Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335344634Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335352984Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335361054Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335371284Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335378984Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335389414Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335396765Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335424635Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335433265Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335439785Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335446195Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335452165Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335458125Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335465755Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335473765Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:34.335547566Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"83b82f3a-8c43-4951-8b91-7c37ab98da25","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized)]","latency_us":474520,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:34.335822869Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.216146361s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"83b82f3a-8c43-4951-8b91-7c37ab98da25\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"83b82f3a-8c43-4951-8b91-7c37ab98da25\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.473764666Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61952"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.473851158Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"6f89b1a1-0110-4a73-95d6-377fc87732ec","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61952","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":56,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.55335118Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.553820665Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.553829745Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.577005888Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.577070289Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.598804305Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.598862366Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.619513871Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.619571491Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.638590127Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.638655158Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.657264609Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.65731954Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.676042582Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.676106803Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.694852225Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.694901126Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.713303515Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.713358065Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.731675753Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.731737604Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.750397216Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.750457876Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.769008207Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.769057037Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.78778123Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.78783069Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.805336259Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.80539217Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.823846179Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.823907Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.841370638Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.841418538Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.859763847Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.859817987Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.878356088Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.878405198Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.895784786Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.895847226Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.914307146Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.914360716Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.932153008Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.932207359Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.932458602Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:37.932602673Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028756635Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028835525Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028845605Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028858696Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028864826Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028880126Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028888076Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028902696Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028924006Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028931846Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028938197Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028945467Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028953357Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028958907Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028964537Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028970317Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028979607Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028986827Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.028992637Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:38.029000117Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"2e1ed4e1-8721-41cb-9661-a4cb2a33540d","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized)]","latency_us":475455,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:38.029204009Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.294551486s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2e1ed4e1-8721-41cb-9661-a4cb2a33540d\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2e1ed4e1-8721-41cb-9661-a4cb2a33540d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.324582195Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.324921288Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.324933208Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.349315545Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.349376946Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.370156221Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.370214842Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.390909797Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.390963518Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.409859132Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.409925313Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.429453734Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.429514445Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.448192177Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.448237168Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.466756268Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.466802908Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.486748385Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.486798945Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.505337856Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.505396346Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.524339381Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.524398102Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.542885472Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.542944512Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.561897188Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.561945618Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.579852161Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.582206998Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.599288922Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.601651568Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.617943064Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.618001654Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.635404651Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.635456872Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.654224855Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.654281946Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.673328102Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.673387082Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.692229856Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.692281497Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.710476034Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.710538524Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.710757267Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.710900778Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.804340029Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.804403019Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.804410809Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80441916Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80442742Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80443299Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80445134Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80445639Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80446316Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80447196Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80447736Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.804482Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80448879Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80450755Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.80451267Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.804517611Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.804525771Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.804530681Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.804535371Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:44.804547191Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"d11ef4ca-027a-4691-a266-cfc9ddcfd52b","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized)]","latency_us":479796,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:44.804728723Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.012813062s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d11ef4ca-027a-4691-a266-cfc9ddcfd52b\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d11ef4ca-027a-4691-a266-cfc9ddcfd52b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:45.681726515Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"7.992700225s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.818168405Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.818468769Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.818478639Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.843092848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.843161939Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.86527436Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.86532858Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.890394145Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.890455706Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.912128592Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.912195512Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.935101852Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.935153063Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.955355042Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.955417963Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.976271629Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.976333Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.996252166Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:48.996312397Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.01858594Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.01865079Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.040190244Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.040247354Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.059817587Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.059869327Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.0812982Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.081344401Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.102613622Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.102677023Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.125036047Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.125088927Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.146308928Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.146358999Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.166114273Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.166166703Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.18879869Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.188867551Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.209395744Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.209450594Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.231466295Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.231517855Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.254185502Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.254252463Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.254466696Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.254592277Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366300945Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366366405Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366373535Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366393746Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366398976Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366404976Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366409396Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366414826Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366424976Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366429856Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366434526Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366440206Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366444976Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366451016Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366454516Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366458776Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366463296Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366468746Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.366473396Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:49.366606518Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"16df19f2-8bc7-49b4-988b-3b448f8ae02d","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized)]","latency_us":548311,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:49.36679595Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.996976475s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"16df19f2-8bc7-49b4-988b-3b448f8ae02d\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"16df19f2-8bc7-49b4-988b-3b448f8ae02d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.364547995Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.364904849Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.364915669Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.38787786Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.387947431Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.408000638Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.408058479Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.426491228Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.426559219Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.444034867Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.444089788Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.46369035Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.463733281Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.481109908Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.481162309Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.498633967Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.498672217Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.51564057Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.515682051Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.533331651Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.533379661Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.554471781Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.554516091Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.572677887Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.572727788Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.591200547Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.591265628Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.609160851Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.609222662Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.626989574Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.627033774Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.645235441Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.645291191Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.663188824Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.663253145Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.681490872Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.683434444Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.699282564Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.699347425Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.718004566Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.718054447Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.735846419Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.73590093Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.736184783Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.736348764Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828393829Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.82845424Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.82846066Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.8284898Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.82849702Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.82850346Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.82850896Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.82851405Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.82852021Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828526521Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828532991Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828539421Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828545801Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828554701Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828559071Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828563191Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828567231Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828571251Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828581831Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:18:56.828633672Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"59388ddb-9659-4630-93d9-48945f1da0b2","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized)]","latency_us":463935,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:18:56.828846754Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"11.632213299s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"59388ddb-9659-4630-93d9-48945f1da0b2\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:18:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"59388ddb-9659-4630-93d9-48945f1da0b2\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:07.507199113Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58880"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:07.507292384Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"8992961c-eb47-4cf1-a9b2-0076f968dec7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58880","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.463055189Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.463352122Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.463368863Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.489230626Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.491953537Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.509325484Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.509383755Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.528996607Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.529063998Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.547523907Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.547586848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.56538882Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.565447921Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.58384701Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.58389333Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.602007286Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.602070926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.619961289Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.62001443Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.638145796Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.638198406Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.656099259Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.65614461Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.674687721Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.674755501Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.69403853Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.69408133Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.710361665Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"18.21793168s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.712165036Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.712171636Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.731534955Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.731590536Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.750015995Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.750069446Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.768169501Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.768233472Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.787283588Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.787327479Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.806209893Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.806253503Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.824273558Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.824317199Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.8429459Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.84300564Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.843327394Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.843502546Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941520218Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941587259Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941597479Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941606339Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.94161193Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941619879Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.94162844Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.94163526Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.94164203Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.94165017Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.94165682Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.94166551Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.94169053Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941699151Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941713151Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941721421Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941728771Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941735031Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.941742241Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:08.941813452Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"7785e919-3c82-4fa1-8175-8b222e0543cc","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized)]","latency_us":478644,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:08.942026554Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"15.685159157s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7785e919-3c82-4fa1-8175-8b222e0543cc\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:19:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7785e919-3c82-4fa1-8175-8b222e0543cc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.629029078Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.629431122Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.629440663Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.651888657Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.651961438Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.672439471Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.672474081Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.693855383Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.693907964Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.713020971Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.713062052Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.732596583Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.732640484Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.75169402Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.75173002Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.770119309Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.770157009Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.789261216Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.789300326Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.807554194Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.807589064Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.827004925Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.827056815Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.846459055Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.846506936Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.864943445Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.864979975Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.883276003Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.883312343Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.901661022Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.901705112Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.920357704Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.920413085Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.938141926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.938175526Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.956555355Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.956603865Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.975728112Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.975767133Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.993921279Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:24.993951239Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:25.012176706Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:25.012221426Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:25.012474749Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:25.012612051Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111500492Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111563053Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111587063Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111594433Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111603303Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111608823Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111615863Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111625243Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111632043Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111639154Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111647544Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111664954Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111673744Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111681084Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111690084Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111697114Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111705654Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111712114Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.111727415Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:25.111816776Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a47ea12b-53eb-401b-a889-8f3c7d1aaac9","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\", TimeNotSynchronized), (\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\", TimeNotSynchronized), (\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\", TimeNotSynchronized), (\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\", TimeNotSynchronized), (\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\", TimeNotSynchronized), (\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\", TimeNotSynchronized), (\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\", TimeNotSynchronized), (\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\", TimeNotSynchronized), (\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\", TimeNotSynchronized), (\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\", TimeNotSynchronized), (\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\", TimeNotSynchronized), (\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\", TimeNotSynchronized), (\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\", TimeNotSynchronized), (\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\", TimeNotSynchronized), (\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\", TimeNotSynchronized), (\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\", TimeNotSynchronized), (\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\", TimeNotSynchronized), (\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\", TimeNotSynchronized)]","latency_us":482593,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:25.112073308Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"14.864171809s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a47ea12b-53eb-401b-a889-8f3c7d1aaac9\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:19:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a47ea12b-53eb-401b-a889-8f3c7d1aaac9\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:37.543599017Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:37.543654048Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"6fde694e-66d1-426d-bd60-1aaa5bf994e3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45215","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":42,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: aa7e09fe-9652-4e85-a32d-c955237752d1, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 640a4d7b-43e4-41e4-914c-798fafc7477b, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 17ff57be-541b-4be0-88ba-76c07e13e827, kind: Service { id: 640a4d7b-43e4-41e4-914c-798fafc7477b }, name: Name(\"ntp-640a4d7b-43e4-41e4-914c-798fafc7477b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 100])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: 06252ed0-79b6-4cf3-a662-7733f01c14de, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: da97951e-47b6-4f52-b7b8-3531696ee4bc, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e37693be-af5f-4e66-b536-8ada15d64fd0, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e517e38c-3218-4379-a038-fee6d2241dcc, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b10b6fc0-2ede-45da-aebf-47851e56f835, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 29c6e5ce-8d24-4f29-8b67-3e0bccf0054e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1329be04-67e5-46a4-a0ac-1869bd290bcf, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f7e6faec-0230-43e2-8146-b612de3e5bbc, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 84dcd2c0-1d9e-4702-8041-046179b53476, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 001c98fd-0e11-4673-a246-07e1d419966c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External } } } }, OmicronZoneConfig { id: beac8070-cca6-4408-8052-ffcd37401e3d, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External } } } }, OmicronZoneConfig { id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 11faa295-2980-402b-8b1f-5dde16195481, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External } } } }, OmicronZoneConfig { id: d08a4654-6158-42b2-8f20-5cbc4e053c56, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External } } } }, OmicronZoneConfig { id: b6db9c49-afad-4e42-bf22-de071daefd9c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:39.980384097Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:39.980732801Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:39.980741221Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7f76f1e3-d67e-418e-855e-dccd3164c0dc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.002616619Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.00266347Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: aa7e09fe-9652-4e85-a32d-c955237752d1, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.024944973Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.024985393Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76cccebb-a2fc-433e-9266-4ea7f002ac0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.043220189Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.04326362Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2c74ddf5-baea-4359-aeb5-d5897afea56f, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.062352376Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.062412537Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1cf97cd5-2bfc-4f9b-9562-4d2722532342, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.080533373Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.080573223Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84bb5ac4-5b69-491a-915b-56f3458c45f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.10056767Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.100606641Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb0a7b7e-c94d-46a1-b575-2f0b132ffd94, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.11992403Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.1199564Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06252ed0-79b6-4cf3-a662-7733f01c14de, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.138085376Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.138121746Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: da97951e-47b6-4f52-b7b8-3531696ee4bc, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.158341505Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.158394056Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e37693be-af5f-4e66-b536-8ada15d64fd0, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.17806824Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.17811492Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f57b1b30-af19-4baf-9c8e-ced74f6b8ce4, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.196568819Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.19660856Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 84dcd2c0-1d9e-4702-8041-046179b53476, dataset_name: DatasetName { pool_name: ZpoolName { id: eaa88cfe-d08c-4ee9-8c11-845f26835ca2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.216632067Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.216669067Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a272be5-b5e7-4093-8bf3-890e350d2da4, dataset_name: DatasetName { pool_name: ZpoolName { id: ad3dbf69-5e12-43e4-b601-7fce4e3f33b5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.235705403Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.235744724Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 001c98fd-0e11-4673-a246-07e1d419966c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b494463-e11e-42a2-b533-5933dfea9e20 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.254351955Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.254400336Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beac8070-cca6-4408-8052-ffcd37401e3d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0112b31e-4dc2-42d6-a774-9294ebc7e996 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.273684814Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.273724355Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f39c12eb-9c91-43c6-8261-cfd82963ca8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2da812e0-f19d-4c88-af4f-92430c5507c5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.29356092Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.29359748Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11faa295-2980-402b-8b1f-5dde16195481, dataset_name: DatasetName { pool_name: ZpoolName { id: 1bbd5d50-8020-48f2-ba68-03b3ab3e35af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.312789037Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.312843825Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d08a4654-6158-42b2-8f20-5cbc4e053c56, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b354c5b-27b9-467d-9c0f-816eaba8dbe6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.331949035Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.331992096Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63, dataset_name: DatasetName { pool_name: ZpoolName { id: 6464a1b3-39ff-4f6d-bb8d-28b483455b0c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.351341106Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.351389066Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b6db9c49-afad-4e42-bf22-de071daefd9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 22b3b804-39bb-4b5d-9386-43e03fd38f0f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.369654603Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.369704454Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.369951457Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.370083518Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.509398399Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.514630768Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.514654469Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.514666499Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.514683419Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.51470434Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.51471158Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.517577422Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.517681703Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.519011668Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.691049781Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.691311833Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.691348294Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.691387444Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.737050372Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.737290475Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.737326926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.737344096Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.748834716Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.7491431Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.755731855Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.761293458Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.766469546Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.771932758Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.776593361Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.78175542Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.923574879Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.928978461Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.933753855Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.938302636Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.942985439Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.94745652Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.955950207Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.962289939Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.967148103Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.972752767Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.979403853Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.985458581Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.990492037Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:40.996573037Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.003047591Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.010641207Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.01792175Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.023967468Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.029686883Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.035658911Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.04262807Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.093041562Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.142971719Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.18718806Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.231808277Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.26471825Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.266392979Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.266645362Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.324550269Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 186, 42]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.32459334Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.326728624Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.326749174Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.384664511Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 171, 233]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.384710182Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.387040508Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.387054719Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.45242005Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 205, 111]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.452473711Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.456101572Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.456117182Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.508349185Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 140, 22]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.508408245Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.511039745Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.511082386Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.561009832Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 182, 18]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.561062943Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.563359879Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.563645832Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.563813754Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.563827024Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.563903155Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.563909035Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.564057137Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.564064167Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.564199949Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.564217399Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.564271169Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.565932538Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.567910361Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.567923141Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.567929421Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.569908463Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.569935244Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.573624405Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.575866991Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.575873821Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.70564179Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.711259387Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.717585909Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.723523337Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.729606715Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.735550383Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.741494061Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.747294126Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.752558276Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:41.758682005Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:41.970390098Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"24.386970422s"} {"msg":"Profile for oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.63153644Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.713716733Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.713745844Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.728164587Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.753835038Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.75660274Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.757606491Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.757985735Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.759333581Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.76015076Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.761377944Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.761653387Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.809426289Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.87562832Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.91260306Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.919343377Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.925172863Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.930489253Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.937985918Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.983411093Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.992442056Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:52.999755769Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:53.007272364Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:53.096181983Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:53.096274604Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:53.107168528Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 62.614627ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.772253265Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 29.694176ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.785635197Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 51.730815ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.799417753Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 42.701932ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.813578314Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 57.550077ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.826101616Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 32.388152ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.840084965Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 50.549409ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.853682949Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 41.359849ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.868296905Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 47.626713ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.883063552Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 55.568506ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.897514416Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 54.104695ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.91198147Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 100.180239ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.925262551Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 141.326134ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.937997106Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 77.532196ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.951002373Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 83.615567ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.962730626Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 55.208083ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.975293819Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 90.718835ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:53.988558989Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 113.296889ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.003234046Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 136.429632ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.016267964Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 80.201806ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.032175344Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 100.366019ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.049053063Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 148.141892ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.06713603Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 71.095655ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.085844453Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 27.832072ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.101529551Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 100.947173ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.120461006Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 203.121166ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.154530342Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 120.212678ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.170150989Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 167.607436ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.23184979Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 73.913285ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.255480368Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 275.617347ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.344792171Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 119.5444ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.362869296Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 235.346299ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.3799105Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 197.363642ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.39583661Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 127.154773ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.410345285Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 102.809165ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.426889283Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 231.94066ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.448145114Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 74.366841ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.464380798Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 256.235491ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.478950183Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 323.094383ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.493243476Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 379.42405ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.509154216Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 216.187131ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.526102838Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 138.622295ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.541342851Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 248.144517ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.557351383Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 424.904054ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.572442324Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 306.436876ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.586211151Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 215.280264ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.602613127Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 523.701473ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.617403745Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 289.232309ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.673117787Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 525.368605ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.709116525Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 271.371984ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.731790003Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 529.839183ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.751883551Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 1.183407124s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.838800157Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 935.237838ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.886340036Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 462.140693ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.937620418Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 706.375529ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.95894487Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 427.178857ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:54.992926246Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 215.965147ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.061229081Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 408.5951ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.089751885Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 611.763381ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.139259786Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 540.476862ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.206200026Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 1.185258599s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.252524512Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 1.083263545s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.272570379Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 695.463132ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.292458515Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 1.013857426s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.309995154Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 785.696489ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.337983891Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 953.447148ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.424303941Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 1.284971653s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.446958918Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 1.00407112s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.515699018Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 1.231817981s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.683152338Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 861.660526ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.764865826Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 1.391929578s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.782572076Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 823.284348ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:55.857430926Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 1.355949811s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.007456958Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 954.628176ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.040551914Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 1.315694575s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.14165953Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 867.763364ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.339637297Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 1.305895611s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.369125692Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 538.06388ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.393521599Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:56.443484275Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 1.169037463s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.455570912Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:56.455903066Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 1.101551559s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.562630978Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 612.847026ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.641377141Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 1.23391446s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.694671946Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 785.560921ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.746559454Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 51.269286ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.81927635Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 124.476666ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.883254206Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 1.049736062s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.93036184Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 784.450811ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:56.94441282Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 1.400750664s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.028468744Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 126.756901ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.042291Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 208.374999ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.182448621Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 1.067873546s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.192709347Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 988.011571ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.218429829Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 889.703146ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.268165993Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 978.37159ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.378428915Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 1.156750027s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.406497083Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 862.992774ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.469363527Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 1.301741988s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.544809173Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 1.0357226s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.638599007Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 1.124577642s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.676992263Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 1.047791111s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.687974117Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 1.373108026s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:57.742207763Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 948.838042ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.025928672Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 1.106753931s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.037352712Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:58.039384255Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:58.041361847Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:58.04780895Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:58.054322424Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 942.720382ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.172283532Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 1.494810001s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.219192924Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 954.076021ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.272701432Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 66.712971ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.28403109Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 73.809304ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.294087184Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 1.204158186s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.344349105Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 143.04724ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.362680383Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 1.280798529s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.374204423Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 114.244948ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.38540511Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 562.332877ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.443800233Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 164.948048ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.514353264Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 229.512548ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.527713737Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 1.136075214s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.576097774Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 954.806067ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.688016704Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 243.256803ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.699861419Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 712.08689ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.789393645Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 265.58997ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.821420998Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 1.20497449s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.858355167Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 894.236766ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.875860676Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 701.383117ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.957836106Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 1.137789812s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:58.990539487Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 1.446156547s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.022126236Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 721.374965ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.100689707Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 996.203541ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.128524342Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 1.487043435s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.141999645Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 1.26547317s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.157443991Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 1.005748411s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.241485014Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:59.24271477Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:19:59.242938029Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 1.151932581s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.516650867Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 835.17181ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.562051702Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 27.930744ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.573205028Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 121.464432ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.618638796Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 687.426734ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.677941767Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 741.477794ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.701577775Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 601.834887ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.719331677Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 1.369558277s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.737757196Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 892.623268ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.753508474Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 148.230583ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.765882545Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 1.309021463s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.785840451Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 947.595042ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:19:59.965963715Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 335.771109ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.008247295Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 1.297639438s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.077696232Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 1.361040926s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.138437692Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 781.231179ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.151362588Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 1.476454801s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.262036394Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 922.974469ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.335396417Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 550.130522ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.357787381Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 1.248973283s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.378266093Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 1.284479507s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.411578681Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 1.159089788s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.438299274Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 1.451830411s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.455822893Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 783.81242ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.4802297Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 1.24407625s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.644685966Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 1.195800784s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.65999159Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 647.813353ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.682999021Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 629.340584ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.921041422Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 1.282560167s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.930988955Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 1.107661346s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:00.945638861Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:01.05121619Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:01.05305735Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 1.010214462s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.110583763Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 1.149666907s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.124501779Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:01.258501772Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:01.258583663Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 974.049916ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.278115154Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 982.469722ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.290571586Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 49.277797ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.31475266Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 1.354930738s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.345950904Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 50.720114ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.379818248Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 594.677756ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.400578604Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 205.654057ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.494216226Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 1.124416042s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.559924112Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 1.236301789s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.585119448Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 924.470364ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.619792731Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 36.757513ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.638541024Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 1.18511835s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.657161026Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 121.821932ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.689400026Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 772.541843ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.710802934Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 460.038857ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.727061949Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 1.454776719s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.751924411Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 238.423187ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.829049346Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 872.600894ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.87259037Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 708.454625ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.905288571Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 1.287376742s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:01.92277904Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 826.952535ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.011770649Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 870.902682ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.072016662Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 326.293762ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.08853374Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 1.362735019s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.136722207Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 1.032328414s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.206110374Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 594.813481ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.233267512Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 1.372923289s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.271049331Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 1.385043661s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.289910935Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 704.109746ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.310751931Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 669.715992ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.432529193Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 647.384766ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.503208755Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 580.823285ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.56624261Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 1.082557052s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.639473792Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 503.482083ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.716136822Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 1.361071674s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.780287131Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 1.103339713s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.804882709Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 1.385190272s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.846329139Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 1.174293207s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.86232633Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 770.588202ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.876493881Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 1.1912276s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.89049726Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 1.315768426s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:02.961163792Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 949.245775ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.035684808Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 636.632315ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.125292184Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 1.472264599s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.171130145Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 1.048898333s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.187106156Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 1.211134103s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.226292191Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 1.009244577s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.243962241Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 704.795075ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.262197278Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 1.207607214s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.278713855Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 548.182416ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.511892611Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 629.726734ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.657897728Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 1.345883085s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.667978592Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 523.83356ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.687758617Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 1.460745664s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.738862227Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 1.040947013s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.776223591Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 507.647861ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.922320509Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 1.399217575s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.979822091Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 900.959424ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:03.997819746Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 1.108961092s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.046618278Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 1.492394317s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.073389882Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 1.043662735s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.095309111Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 1.177554145s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.152117316Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 688.20675ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.224679949Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 1.368443905s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.245684427Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 688.850611ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.258418682Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 972.518123ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.267874939Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 726.980455ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.288721096Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 587.083871ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.298237954Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 932.615084ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.443022577Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 969.748824ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.453774019Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 534.962615ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.500017953Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 701.574336ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.658791185Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 618.758263ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.831714577Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 820.060397ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.903068927Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 1.165436398s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.91658212Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 1.073243369s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.928100711Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 721.410354ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:04.960877563Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 1.037804928s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.029881536Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 532.805794ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.045493532Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 606.689878ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.060909787Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 1.252674113s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.154245786Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 1.137688854s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.168374957Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 1.131373094s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.212175134Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 1.267509119s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.256531367Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 1.312586341s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.347455159Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 1.457897559s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.374796889Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 1.209626835s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.38808204Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 1.383778542s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.402309861Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 818.971644ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.437250888Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 1.208783772s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.463166542Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 1.448149968s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.585867134Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 1.354060935s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.600628992Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 518.347423ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.631050367Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 748.008642ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.682058066Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 1.172507748s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.695475478Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 796.796879ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:05.737027009Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 1.104603425s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:06.018106559Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 1.260623826s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:06.079668207Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 1.35497514s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:06.100012198Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 1.387515123s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:06.162214254Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 611.185582ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:06.268770533Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 609.179811ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:06.320333348Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 1.270648065s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:06.355027015Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 744.3002ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.058099439Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 1.255081522s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.068926412Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 1.129137946s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.076865452Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 945.285856ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.085401839Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 1.167718305s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.09434049Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 1.354932019s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.101723074Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 833.624689ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.109487102Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 756.126086ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.117040778Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 943.025448ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.123868495Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 1.394358911s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.13048063Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 1.480194687s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.138677053Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 591.199536ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.146508122Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 745.282737ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.153785275Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 1.494053279s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.161439211Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 742.15866ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.169783736Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 1.128439787s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.350594518Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 1.323780137s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.465837276Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 1.324776914s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.558348495Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 1.380119681s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.634898752Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:07.738453679Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52282"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 756.416211ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.773371626Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 1.457229706s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.836578233Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:07.865177577Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"937a7480-b1f9-4d31-abe7-fa468fe731ca","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52282","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":89929,"response_code":"200"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 651.027538ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.883762238Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 768.685758ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.909275288Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 1.112663304s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.924299228Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 717.660842ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:07.953096235Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 821.069916ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.040883461Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 1.141625283s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.076143431Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 1.344772884s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.216557995Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 747.606386ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.273905605Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 878.645271ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.334453732Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 1.22718982s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.471472237Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 879.996287ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.489678584Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 648.907539ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.536266932Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 993.899405ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.544693048Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 684.751011ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.553228995Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 818.627004ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.627591469Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 1.255065196s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.665255756Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 1.339830314s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.67881103Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 1.297737051s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.686490507Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 1.275992365s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.800627892Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 1.434124779s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.871168203Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 1.19866038s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:08.893308424Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 712.107599ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.025216081Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 740.929134ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.033692387Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 1.186952512s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.046072037Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 1.320133291s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.195754595Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 1.173460225s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.223332828Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 784.499547ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.232626194Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 529.025966ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.247966608Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 562.493789ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.301832029Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 1.24202291s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.380478351Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 987.354687ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.45706128Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 1.095757559s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.548373417Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 957.935341ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.570873282Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 1.054852198s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.709306933Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d\") failed: Property not found. retry in 1.037661251s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.748015492Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 737.199634ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.784209503Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 1.110858865s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.793065163Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 664.017258ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.874328586Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 931.707075ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.931178611Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 862.425823ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:09.994140155Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 502.409096ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.026962678Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e\") failed: Property not found. retry in 1.050598165s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.035762207Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0\") failed: Property not found. retry in 982.708217ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.087068189Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc\") failed: Property not found. retry in 1.281419707s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.101071528Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c\") failed: Property not found. retry in 1.108898176s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.24315643Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf\") failed: Property not found. retry in 1.080134936s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.315094326Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c\") failed: Property not found. retry in 925.509158ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.40616758Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc\") failed: Property not found. retry in 1.353762483s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.453933952Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481\") failed: Property not found. retry in 972.091868ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.527325504Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476\") failed: Property not found. retry in 578.984879ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.5366171Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835\") failed: Property not found. retry in 884.033686ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.545549721Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 668.672069ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.553754024Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c\") failed: Property not found. retry in 1.265590711s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.562095649Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\") failed: Property not found. retry in 1.244817383s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.638749639Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc\") failed: Property not found. retry in 865.670576ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.656115366Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 769.332786ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.774467719Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4\") failed: Property not found. retry in 745.512324ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.987363505Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56\") failed: Property not found. retry in 1.431570945s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:10.995802651Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63\") failed: Property not found. retry in 791.985061ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:11.004395108Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 1.390012977s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:11.232102981Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 779.273418ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:11.609536914Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23\") failed: Property not found. retry in 1.054964753s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:12.400148746Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4\") failed: Property not found. retry in 655.668063ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:12.634708887Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501775675Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_f39c12eb-9c91-43c6-8261-cfd82963ca8c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501853627Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_b6db9c49-afad-4e42-bf22-de071daefd9c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501861556Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_001c98fd-0e11-4673-a246-07e1d419966c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501867767Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_b10b6fc0-2ede-45da-aebf-47851e56f835"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501873627Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_1329be04-67e5-46a4-a0ac-1869bd290bcf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501879357Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_84dcd2c0-1d9e-4702-8041-046179b53476"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501884537Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_9a5c21b7-6d47-4773-9ae5-0cfab0623b23"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501889637Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_29c6e5ce-8d24-4f29-8b67-3e0bccf0054e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501895287Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_5440be3c-47a7-4c5f-bac9-47f00b0ba6eb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501904057Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_e517e38c-3218-4379-a038-fee6d2241dcc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501910097Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_f57b1b30-af19-4baf-9c8e-ced74f6b8ce4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501916247Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_11faa295-2980-402b-8b1f-5dde16195481"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501921247Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_f7e6faec-0230-43e2-8146-b612de3e5bbc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501940837Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_e37693be-af5f-4e66-b536-8ada15d64fd0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501949888Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_beac8070-cca6-4408-8052-ffcd37401e3d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501956488Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_5a272be5-b5e7-4093-8bf3-890e350d2da4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501961838Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d08a4654-6158-42b2-8f20-5cbc4e053c56"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501966658Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_da97951e-47b6-4f52-b7b8-3531696ee4bc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.501980138Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_ad1eb1aa-b6cd-42eb-81aa-baa08dd29c63"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.505579363Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.531487783Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.60001386Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"48ad40de-eca5-4495-9ea6-f2b01898bfbe","remote_addr":"[fd00:1122:3344:101::1]:56214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":33618341,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.720814061Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1212"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.723031236Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.731934127Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.767772034Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:679"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.909136198Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:692"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:13.980299036Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:803"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:16.376066929Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:16.409349647Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"34.709164395s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:16.509531044Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:16.639050993Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:16.800429375Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:16.93481688Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:17.144027193Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:17.429479672Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:17.719051518Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:18.000494362Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:18.633531155Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:19.207295555Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:20.033901095Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:20.418304396Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:21.165849178Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:22.569220092Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:22.574577023Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:20:22.910360373Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:20:23.932210328Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:24.106352423Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:24.273774713Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:20:24.935258559Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/3f09cc19-1113-4b10-8b38-5af9a34e8833/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:25.692774344Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-01T00:20:25.938847267Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/9f9b1e33-4da8-44e7-8e80-581afd22135c): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:20:26.10578674Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78a35ffc-b3b4-4f4b-9be3-0dfa74d85fd4\", \"content-length\": \"123\", \"date\": \"Wed, 01 May 2024 00:20:27 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"rack not found\", request_id: \"78a35ffc-b3b4-4f4b-9be3-0dfa74d85fd4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:27.134325852Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4b604cbc-103c-4709-a5da-5820313a46aa\", \"content-length\": \"124\", \"date\": \"Wed, 01 May 2024 00:20:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4b604cbc-103c-4709-a5da-5820313a46aa\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:29.055966486Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:30.188563748Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:30.333181478Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:50606"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:30.340799505Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:30.420159496Z","hostname":"buskin","pid":1590,"uri":"/inventory","method":"GET","req_id":"0d90cc76-09fa-402e-b76d-20bf865a0cbc","remote_addr":"[fd00:1122:3344:101::a]:50606","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":86272,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:30.421958256Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:30.422445871Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:30.432938541Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:30.435966335Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"1607448f-1751-4929-a8cf-ee61e8be30bc","remote_addr":"[fd00:1122:3344:101::a]:50606","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":14667,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:32.758816083Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:64997"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:32.762850588Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b44f2c8a-be54-41f2-b112-4e20be72a7f8","remote_addr":"[fd00:1122:3344:101::a]:64997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4013,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.554010516Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:43830"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.566870092Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03536075-d04e-4a60-ace2-84f1c9f1b87b, kind: Service { id: e37693be-af5f-4e66-b536-8ada15d64fd0 }, name: Name(\"external-dns-e37693be-af5f-4e66-b536-8ada15d64fd0\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e9716f54-d399-42dd-8291-09993d421aed, kind: Service { id: da97951e-47b6-4f52-b7b8-3531696ee4bc }, name: Name(\"external-dns-da97951e-47b6-4f52-b7b8-3531696ee4bc\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 434b2a3d-59d1-4048-acae-1ffba7ebee70, kind: Service { id: 5440be3c-47a7-4c5f-bac9-47f00b0ba6eb }, name: Name(\"nexus-5440be3c-47a7-4c5f-bac9-47f00b0ba6eb\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5512139c-ccb1-43d8-bdb6-0c870533b192, kind: Service { id: e517e38c-3218-4379-a038-fee6d2241dcc }, name: Name(\"nexus-e517e38c-3218-4379-a038-fee6d2241dcc\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 42])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e10fa3ca-1988-4349-ab1c-5e0f581433d5, kind: Service { id: 9a5c21b7-6d47-4773-9ae5-0cfab0623b23 }, name: Name(\"nexus-9a5c21b7-6d47-4773-9ae5-0cfab0623b23\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.568659023Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.569664344Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.569680874Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.569696254Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.569710704Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.569727785Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.569888397Z","hostname":"buskin","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"749e930c-9fe0-45aa-9e01-3a8dbf70d73f","remote_addr":"[fd00:1122:3344:101::a]:43830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":15839,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.571420284Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:865"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.714040142Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60568"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.714845018Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.801625976Z","hostname":"buskin","pid":1590,"uri":"/inventory","method":"GET","req_id":"6fb8b45c-eada-44b3-9512-ebaab92f2b44","remote_addr":"[fd00:1122:3344:101::a]:60568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87503,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.80195566Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.802026331Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.802358735Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:33.803298815Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"737abf6a-a184-4fbd-ac54-74dd553c9eb2","remote_addr":"[fd00:1122:3344:101::a]:60568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1450,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:37.901569478Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60028"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:37.901678809Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"c9301838-b0ea-494b-b0ba-be225b1efd1c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60028","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.232231921Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62237"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.317509079Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e57de68c-7858-4514-9fed-c56ac6060aec","remote_addr":"[fd00:1122:3344:101::a]:62237","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":85216,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.31761782Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:56208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.370433059Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"afdbdd91-5a39-43b2-9760-538971c1141d","remote_addr":"[fd00:1122:3344:101::c]:56208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52752,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.370680992Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42144"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.370740263Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e3ea7311-a3e9-4198-93b1-db44fba1f37b","remote_addr":"[fd00:1122:3344:101::a]:62237","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":113,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.370931395Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"777a38f4-1926-44fa-a871-12200f942863","remote_addr":"[fd00:1122:3344:101::c]:56208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":153,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.405423996Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.406544039Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.406915233Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.408107917Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.408131447Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"PUT","req_id":"fd017240-dd80-4565-b961-710f3aa8d3b0","remote_addr":"[fd00:1122:3344:101::a]:62237","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7358,"response_code":"204"} {"msg":"Attempting to update network config with generation: 3, current_generation: 2","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.413064573Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.413156554Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/49112153-160e-48f4-abfa-c3642e83cda8/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.413437377Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 3","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.413647239Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.41370015Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"PUT","req_id":"ef1436c9-73a3-4252-a312-97614caffdd5","remote_addr":"[fd00:1122:3344:101::c]:56208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1336,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.421159715Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"61ee2a9f-89d5-45b5-bb7c-d305454d6cc0","remote_addr":"[fd00:1122:3344:101::b]:42144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50348,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:20:57.421548039Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a6975dab-6a40-4a2b-b5af-5bddddb0c89f","remote_addr":"[fd00:1122:3344:101::b]:42144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:07.955125453Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42084"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:07.955679519Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"8e223d36-b0dc-48ba-9015-57ce06cbf596","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42084","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":54,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:21.228708279Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:63035"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:21.245493149Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"98772c71-5101-46e5-8b6b-f6cb7b0829b7","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":13410,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:27.365776165Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:62722"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:27.416200858Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"f3b523b2-1d96-4d61-bfac-f3d44a5631c7","remote_addr":"[fd00:1122:3344:101::c]:62722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50357,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:27.41644842Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41799"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:27.416514501Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"bfa9ebb8-88a5-4406-8a44-2b7957713a5b","remote_addr":"[fd00:1122:3344:101::c]:62722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":125,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:27.463625976Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6e758780-290c-42da-8303-7bbbc4c2edf0","remote_addr":"[fd00:1122:3344:101::a]:41799","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47051,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:27.464544116Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"884e8e9c-327f-4f7e-91df-e3654b412609","remote_addr":"[fd00:1122:3344:101::a]:41799","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":722,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:27.630621371Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:50665"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:27.676452381Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0ab3e91a-a7d2-430f-932c-2595aec2a878","remote_addr":"[fd00:1122:3344:101::b]:50665","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45778,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:27.676713994Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"ee6196c7-f97e-4d67-81c7-7f375b4e5115","remote_addr":"[fd00:1122:3344:101::b]:50665","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:37.990495473Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48245"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:37.990585754Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"de415e7a-e3b6-4919-8373-95346d31b477","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48245","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":67,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:51.227469652Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"995038b2-91fe-4e0a-b447-036fb21d8192","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":994,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:57.127685511Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41561"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:57.18223389Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"48185520-3790-4e02-a2ab-d7ff72c594e7","remote_addr":"[fd00:1122:3344:101::a]:41561","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":54499,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:57.182412232Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:39937"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:57.187825234Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7cc0e017-8ad6-410b-af5a-f3ceb2e151c8","remote_addr":"[fd00:1122:3344:101::a]:41561","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4593,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:57.23151999Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0c516edf-8e2f-4400-8abd-f5a84acb6818","remote_addr":"[fd00:1122:3344:101::c]:39937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48891,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:57.231902744Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"da7bda73-f574-46cc-98c7-2db6f8c57165","remote_addr":"[fd00:1122:3344:101::c]:39937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:57.39191096Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42102"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:57.436586317Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"75011413-7c78-4ab9-88e5-e4c5b9b357f0","remote_addr":"[fd00:1122:3344:101::b]:42102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44615,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:21:57.437432016Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"28edb57e-4e0b-41de-9e16-9b3b4fa11ae2","remote_addr":"[fd00:1122:3344:101::b]:42102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":682,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:08.022870007Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:08.023669468Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"6601b53a-3b6f-4bf3-93ff-acbfa49f2231","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34142","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":722,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:21.228752243Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"10bcd477-0d66-4230-bb57-92d57063e625","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":804,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:23.346318751Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:27.337442388Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33731"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:27.386673866Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"9db3b7d0-7ef9-4265-bf68-56f047023739","remote_addr":"[fd00:1122:3344:101::a]:33731","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49174,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:27.386915929Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f67ad831-a85b-4a29-8507-c97cc8dfc0c8","remote_addr":"[fd00:1122:3344:101::a]:33731","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:27.616570255Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:41328"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:27.665397309Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"29f1a9eb-0dd6-4718-b031-03a6a687ac70","remote_addr":"[fd00:1122:3344:101::c]:41328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48772,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:27.665654332Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"3c484489-c98b-4439-93ce-cbae25417794","remote_addr":"[fd00:1122:3344:101::c]:41328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:27.716276687Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:49853"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:27.763850217Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"967bcee9-a1aa-48e0-9c8e-0a6c816cc5b1","remote_addr":"[fd00:1122:3344:101::b]:49853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47464,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:27.76411268Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"d782b251-f24b-48fd-bdba-a4579ce9c7ce","remote_addr":"[fd00:1122:3344:101::b]:49853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:38.054453791Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45484"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:38.054559702Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"21a9e7f6-b5f2-48db-bea6-40683136a682","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45484","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":73,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:51.227609147Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"8702457a-58c4-4575-82df-ec9dc7f9f299","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":479,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:57.154968626Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:57.204823002Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"31fc7d96-f68e-4523-8385-7e6f27dd8fef","remote_addr":"[fd00:1122:3344:101::a]:55179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49778,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:57.204979874Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:45712"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:57.209149151Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"1a81bf04-6cce-473b-980d-84bd3d4d0165","remote_addr":"[fd00:1122:3344:101::a]:55179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3997,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:57.250981116Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"bfe719b8-2a95-4659-a306-6ec4537bd725","remote_addr":"[fd00:1122:3344:101::c]:45712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45919,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:57.251293909Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8cd651a6-29e3-4fc2-9205-a67ece327e03","remote_addr":"[fd00:1122:3344:101::c]:45712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:57.30378542Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:53942"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:57.351002651Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"1acca554-c7f0-406d-8e6a-f0f4c2fe15f9","remote_addr":"[fd00:1122:3344:101::b]:53942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47136,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:22:57.351241303Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"73bc6f0f-78df-4dee-bfcf-7f43ae871798","remote_addr":"[fd00:1122:3344:101::b]:53942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:08.085870059Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:32796"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:08.08593628Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"0101a81f-7340-4425-9bf7-026056d7c9e1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:32796","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":62,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:21.244052626Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"2e8fc59b-d814-4942-a9fa-e024945c335f","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":284,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:27.395526217Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:56828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:27.443580462Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6e3a1768-c6cf-400a-9586-c46603812b8f","remote_addr":"[fd00:1122:3344:101::c]:56828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48020,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:27.443694444Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:65141"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:27.447466566Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"71efb01c-547b-4cef-951a-95033608f082","remote_addr":"[fd00:1122:3344:101::c]:56828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3532,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:27.492301605Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6af6a036-1f12-450f-90ae-ff8164f77aaf","remote_addr":"[fd00:1122:3344:101::a]:65141","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48528,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:27.493148015Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"ab05cd78-479a-4f5d-80b4-c1cb2f33dac6","remote_addr":"[fd00:1122:3344:101::a]:65141","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":663,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:27.717331229Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52335"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:27.766620718Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b5deeb25-370d-43d3-b23d-1b9a7aa9ae49","remote_addr":"[fd00:1122:3344:101::b]:52335","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49230,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:27.767012252Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"0f6bb048-bc25-4118-a18b-49d5cbea6229","remote_addr":"[fd00:1122:3344:101::b]:52335","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":184,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:38.118713235Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58970"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:38.118835016Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b56d6bab-701b-4f38-a0cd-543b8af64a65","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58970","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:51.292187406Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"3f19b5bd-ae2e-4616-a057-73f18bc428c0","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":773,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:57.052992094Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:47550"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:57.098775643Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"9dc33d22-2739-409e-96c1-7fec8fa79d6e","remote_addr":"[fd00:1122:3344:101::c]:47550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45718,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:57.099053646Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"fa6b8559-7208-485b-96fb-17e829a7ab3f","remote_addr":"[fd00:1122:3344:101::c]:47550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:57.231322997Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44837"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:57.279060869Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0846493c-7d7d-4989-8238-1d6fdb3be683","remote_addr":"[fd00:1122:3344:101::a]:44837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47652,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:57.279308212Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"6dcccae3-0b6b-4941-8441-26e47fa60992","remote_addr":"[fd00:1122:3344:101::a]:44837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:57.376787758Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42870"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:57.42457695Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"93eb7390-a92d-48e6-9713-e254c32897e9","remote_addr":"[fd00:1122:3344:101::b]:42870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47733,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:23:57.424865954Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"ade93564-a635-4596-ac2a-973c2716d9ac","remote_addr":"[fd00:1122:3344:101::b]:42870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:08.150031873Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:08.150136074Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b42d6292-db3b-4d70-bc0d-d0df0305f5b6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54140","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":79,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:21.292502641Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"5a1933c5-f8e2-4e11-8577-e7f5bb7fd4ee","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":499,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:27.334140705Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:64413"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:27.378509419Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"7758db6e-433b-4df5-b3cb-e97613c46aa2","remote_addr":"[fd00:1122:3344:101::c]:64413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44295,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:27.379519865Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"6a0db5cb-7336-4014-9a8e-0335ebf0f9ae","remote_addr":"[fd00:1122:3344:101::c]:64413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":912,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:27.479841649Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56000"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:27.528586502Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"947bc67d-c8ae-4fc3-812f-870207736ff2","remote_addr":"[fd00:1122:3344:101::a]:56000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48658,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:27.528850255Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"175cb644-8adf-4ed5-9da2-76fe3b0eb749","remote_addr":"[fd00:1122:3344:101::a]:56000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:27.643114882Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:40526"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:27.689194274Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0eb60d05-aba4-43ac-b1c3-151ce5e48ad7","remote_addr":"[fd00:1122:3344:101::b]:40526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46001,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:27.689430787Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"4d3c0af7-82b7-4051-8be0-4c5e47db0880","remote_addr":"[fd00:1122:3344:101::b]:40526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:38.182778727Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:38.182858328Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"deb58ec4-62be-4ac0-8386-3c9719df6b2d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38396","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":68,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:51.293084199Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"b339a78c-6935-4ca5-8856-8a1a94bbeb80","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":466,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:57.063513177Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:58950"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:57.113598855Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"4a052762-277d-4247-b925-49c9f7552964","remote_addr":"[fd00:1122:3344:101::c]:58950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50021,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:57.113860808Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"50bc3b7b-3c45-44df-a50a-5ddb7ad71002","remote_addr":"[fd00:1122:3344:101::c]:58950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:57.262631097Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63597"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:57.311528561Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"738bf361-57d5-47c8-8600-7c1068bfda82","remote_addr":"[fd00:1122:3344:101::a]:63597","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48814,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:57.311745894Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c793f7e6-d8c2-46fa-a760-bc5bd7011dcb","remote_addr":"[fd00:1122:3344:101::a]:63597","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:57.343728767Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:59519"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:57.389069631Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"f8c29dfb-c1f5-4394-a9cb-603abe1be5c0","remote_addr":"[fd00:1122:3344:101::b]:59519","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45245,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:24:57.389414255Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"826ab801-423a-4e40-85a8-d22d760e6d59","remote_addr":"[fd00:1122:3344:101::b]:59519","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:08.21639218Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64570"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:08.216474331Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"ec60250d-7d19-42cf-90f5-97367d7b87a3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64570","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":67,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:21.2929018Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"e5ec20b0-5559-493c-bf36-486ac9b81fbf","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":548,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:27.396945472Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:27.449955864Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a6c9e22b-c1ea-449c-b173-2e1e237f8ec9","remote_addr":"[fd00:1122:3344:101::a]:40710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52947,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:27.450074045Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37129"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:27.453470634Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c98024f1-525a-46ca-8bdf-2dd98cdd9c0c","remote_addr":"[fd00:1122:3344:101::a]:40710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3172,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:27.498237662Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0a970dcc-cbb8-43bf-9925-968fe1c64da0","remote_addr":"[fd00:1122:3344:101::c]:37129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48066,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:27.498665967Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"5b9c5eca-011d-4503-8b2b-eb5afe50722c","remote_addr":"[fd00:1122:3344:101::c]:37129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:27.51484371Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:49122"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:27.56148522Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"98a0518a-43a6-4593-a21a-cc1f37acc215","remote_addr":"[fd00:1122:3344:101::b]:49122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46577,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:27.561761923Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c58f63f2-4f06-40ef-a6bf-ca0646d758c7","remote_addr":"[fd00:1122:3344:101::b]:49122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:38.250937742Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38187"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:38.251014653Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"55c86c14-d022-4318-afd1-85001ff7a277","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38187","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":73,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:51.29375157Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"43cb6469-bcc1-4eb6-90dd-53f2cc8ea7ed","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":708,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:57.075033991Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:45799"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:57.124950577Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b84432b7-d8a3-4780-a082-27f5b35fa7c0","remote_addr":"[fd00:1122:3344:101::c]:45799","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49899,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:57.125331562Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"fc637b9a-97dd-4f83-9d29-38c652535d31","remote_addr":"[fd00:1122:3344:101::c]:45799","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:57.154692135Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60192"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:57.202438357Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"2bb65b70-49a7-4154-888e-89ca2f71c44b","remote_addr":"[fd00:1122:3344:101::a]:60192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47657,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:57.203402638Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"1c97d0e3-f164-4f89-9407-40116e1ae7ee","remote_addr":"[fd00:1122:3344:101::a]:60192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":815,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:57.36487709Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:49018"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:57.410229045Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6107e4bb-8fab-487a-b957-7dd1b6d1885b","remote_addr":"[fd00:1122:3344:101::b]:49018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45272,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:25:57.41065506Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e2276ec6-d6e0-4b26-81a4-a73d572a1a96","remote_addr":"[fd00:1122:3344:101::b]:49018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":178,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:08.284978701Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:08.285477146Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"4d75cb5d-3dd3-44f9-8595-62318f162822","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35877","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.612846489Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63645"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.643526457Z","hostname":"buskin","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: f14c14db-9a38-4635-a931-5bcf0af23dfa, project_id: 19df3dec-dc1d-4092-9b44-41f78e59c03f }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-01T00:26:19.062031Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(c024bad3-d47c-45d2-898f-93d15935eabc), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T00:26:19.108847910Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 41785a54-3052-49f4-965a-a32a419f7c9f, kind: Instance { id: aa6f5f61-878c-4f0f-8c90-b19dbdb2d164 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 28, 137])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(490906), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 41785a54-3052-49f4-965a-a32a419f7c9f, kind: Instance { id: aa6f5f61-878c-4f0f-8c90-b19dbdb2d164 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 28, 137])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(490906), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 41785a54-3052-49f4-965a-a32a419f7c9f, kind: Instance { id: aa6f5f61-878c-4f0f-8c90-b19dbdb2d164 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 28, 137])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(490906), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(490906))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 41785a54-3052-49f4-965a-a32a419f7c9f, kind: Instance { id: aa6f5f61-878c-4f0f-8c90-b19dbdb2d164 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 28, 137])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(490906), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-d32ceb2d022\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 5886b49e-2ca4-4f44-9431-b498bdc8db5c, read_only_parent: Some(Volume { block_size: 512, id: d53e72d1-28b9-49da-97c2-66accca3a611, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ce55748e-24e4-491f-bfb9-db9a6cb104b1, key: Some(\"L0CfpoyE/f+drEYHEcPgHSdWkB+8a++854pj2ubniQA=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::16]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5886b49e-2ca4-4f44-9431-b498bdc8db5c, key: Some(\"CWU1SobL4bFoS7KKDVydGQLQtBEcoCF6YqjjQlPXf7c=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::17]:19003\", \"[fd00:1122:3344:101::13]:19001\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACdSQOhWKFYAABJA6FYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAnUkDoVihWAAASQOhWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJhYTZmNWY2MS04NzhjLTRmMGYtOGM5MC1iMTlkYmRiMmQxNjQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSWdtUHhnNWNkYTI5YThLUXhiaFNIaXdKS2JIUmFYNFQ5WFREY0hBQlcvRSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.643646779Z","hostname":"buskin","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.645073775Z","hostname":"buskin","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 41785a54-3052-49f4-965a-a32a419f7c9f, kind: Instance { id: aa6f5f61-878c-4f0f-8c90-b19dbdb2d164 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 28, 137])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(490906), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 41785a54-3052-49f4-965a-a32a419f7c9f, kind: Instance { id: aa6f5f61-878c-4f0f-8c90-b19dbdb2d164 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 28, 137])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(490906), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 41785a54-3052-49f4-965a-a32a419f7c9f, kind: Instance { id: aa6f5f61-878c-4f0f-8c90-b19dbdb2d164 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 28, 137])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(490906), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(490906))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 41785a54-3052-49f4-965a-a32a419f7c9f, kind: Instance { id: aa6f5f61-878c-4f0f-8c90-b19dbdb2d164 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 28, 137])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(490906), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-d32ceb2d022\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 5886b49e-2ca4-4f44-9431-b498bdc8db5c, read_only_parent: Some(Volume { block_size: 512, id: d53e72d1-28b9-49da-97c2-66accca3a611, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ce55748e-24e4-491f-bfb9-db9a6cb104b1, key: Some(\"L0CfpoyE/f+drEYHEcPgHSdWkB+8a++854pj2ubniQA=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::16]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5886b49e-2ca4-4f44-9431-b498bdc8db5c, key: Some(\"CWU1SobL4bFoS7KKDVydGQLQtBEcoCF6YqjjQlPXf7c=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::17]:19003\", \"[fd00:1122:3344:101::13]:19001\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACdSQOhWKFYAABJA6FYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAnUkDoVihWAAASQOhWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJhYTZmNWY2MS04NzhjLTRmMGYtOGM5MC1iMTlkYmRiMmQxNjQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSWdtUHhnNWNkYTI5YThLUXhiaFNIaXdKS2JIUmFYNFQ5WFREY0hBQlcvRSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(c024bad3-d47c-45d2-898f-93d15935eabc), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T00:26:19.108847910Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-01T00:26:19.062031Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.648464721Z","hostname":"buskin","pid":1590,"uri":"/instances/aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","method":"PUT","req_id":"42efd0d5-f981-46a6-a94b-ec56d28d74a4","remote_addr":"[fd00:1122:3344:101::a]:63645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":35635,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.774762504Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.778050714Z","hostname":"buskin","pid":1590,"uri":"/instances/aa6f5f61-878c-4f0f-8c90-b19dbdb2d164/state","method":"PUT","req_id":"f7a85a8d-0495-4ae8-ba03-ffeefc3a6fa9","remote_addr":"[fd00:1122:3344:101::a]:44215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3193,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.803384991Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 254, 28, 137]), slot: 0, vni: Vni { inner: 490906 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.804061749Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.837196625Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:19.859328396Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:21.376686593Z","hostname":"buskin","pid":1590,"uri":"//9f9b1e33-4da8-44e7-8e80-581afd22135c","method":"GET","req_id":"45c02cac-5a40-4f6c-9a95-c45f65035058","remote_addr":"[fd00:1122:3344:101::d]:63035","local_addr":"[fd00:1122:3344:101::1]:49879","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"9f9b1e33-4da8-44e7-8e80-581afd22135c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":671,"response_code":"200"} {"msg":"Profile for oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:21.675754357Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:21.679345077Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\") failed: Property not found. retry in 62.28956ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:21.869121271Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\") failed: Property not found. retry in 72.172448ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:21.954598581Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\") failed: Property not found. retry in 137.158429ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:22.039009009Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\") failed: Property not found. retry in 579.737965ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:22.188432123Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\") failed: Property not found. retry in 905.973488ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:22.780987067Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\") failed: Property not found. retry in 1.21635506s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:23.698422799Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\") failed: Property not found. retry in 1.208093242s","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:24.924864504Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\") failed: Property not found. retry in 738.359082ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:26.14420405Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:26.913130475Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"wait for service Some(\"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\") failed: Property not found. retry in 29.841241ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:26.922513262Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\") failed: Property not found. retry in 83.865705ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:26.961838418Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.055696523Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.09426403Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACdSQOhWKFYAABJA6FYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAnUkDoVihWAAASQOhWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJhYTZmNWY2MS04NzhjLTRmMGYtOGM5MC1iMTlkYmRiMmQxNjQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSWdtUHhnNWNkYTI5YThLUXhiaFNIaXdKS2JIUmFYNFQ5WFREY0hBQlcvRSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-d32ceb2d022\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 5886b49e-2ca4-4f44-9431-b498bdc8db5c, read_only_parent: Some(Volume { block_size: 512, id: d53e72d1-28b9-49da-97c2-66accca3a611, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ce55748e-24e4-491f-bfb9-db9a6cb104b1, key: Some(\"L0CfpoyE/f+drEYHEcPgHSdWkB+8a++854pj2ubniQA=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::16]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5886b49e-2ca4-4f44-9431-b498bdc8db5c, key: Some(\"CWU1SobL4bFoS7KKDVydGQLQtBEcoCF6YqjjQlPXf7c=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::17]:19003\", \"[fd00:1122:3344:101::13]:19001\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: aa6f5f61-878c-4f0f-8c90-b19dbdb2d164, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 19df3dec-dc1d-4092-9b44-41f78e59c03f, silo_id: f14c14db-9a38-4635-a931-5bcf0af23dfa }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.096121751Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.214500034Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-05-01T00:26:27.215853940Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.21588934Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.215913801Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-01T00:26:27.215853940Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(c024bad3-d47c-45d2-898f-93d15935eabc), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T00:26:19.108847910Z }","propolis_id":"c024bad3-d47c-45d2-898f-93d15935eabc"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.216016032Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(c024bad3-d47c-45d2-898f-93d15935eabc), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T00:26:19.108847910Z }, propolis_id: c024bad3-d47c-45d2-898f-93d15935eabc, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-01T00:26:27.215853940Z } }","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-05-01T00:26:27.270088115Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.270106176Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.270119876Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-01T00:26:27.270088115Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(c024bad3-d47c-45d2-898f-93d15935eabc), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T00:26:19.108847910Z }","propolis_id":"c024bad3-d47c-45d2-898f-93d15935eabc"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.270128596Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(c024bad3-d47c-45d2-898f-93d15935eabc), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T00:26:19.108847910Z }, propolis_id: c024bad3-d47c-45d2-898f-93d15935eabc, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-01T00:26:27.270088115Z } }","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.365823872Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59006"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.420257359Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6f70dfaf-2d3f-46e5-beb5-0a508cc0dbbb","remote_addr":"[fd00:1122:3344:101::a]:59006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":54355,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.420636364Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8b69d2fc-0efe-4160-8d4d-84ba791f14e3","remote_addr":"[fd00:1122:3344:101::a]:59006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.556101981Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:59936"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.605742714Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a817c407-9e12-4373-a5d2-f386d3bfc052","remote_addr":"[fd00:1122:3344:101::c]:59936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49556,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.606012287Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e06c5925-646d-4180-b64d-fe6f0e21937b","remote_addr":"[fd00:1122:3344:101::c]:59936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.65123921Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:55945"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.698150483Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"5ab36e01-d40a-490f-9b6d-66379ea2e5fd","remote_addr":"[fd00:1122:3344:101::b]:55945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46200,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:27.698391905Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7e35ee42-b4ec-479c-b2b5-4cb3e7e51387","remote_addr":"[fd00:1122:3344:101::b]:55945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.461753555Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58669"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.46306393Z","hostname":"buskin","pid":1590,"uri":"/instances/aa6f5f61-878c-4f0f-8c90-b19dbdb2d164/state","method":"PUT","req_id":"19fa6055-51fa-4c82-bf5d-56ca6bc48ecf","remote_addr":"[fd00:1122:3344:101::a]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1246,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-05-01T00:26:36.463168491Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.463196021Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.463217231Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-01T00:26:36.463168491Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(c024bad3-d47c-45d2-898f-93d15935eabc), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T00:26:19.108847910Z }","propolis_id":"c024bad3-d47c-45d2-898f-93d15935eabc"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.463245961Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(c024bad3-d47c-45d2-898f-93d15935eabc), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-01T00:26:19.108847910Z }, propolis_id: c024bad3-d47c-45d2-898f-93d15935eabc, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-01T00:26:36.463168491Z } }","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-05-01T00:26:36.482220057Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.482238827Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.482260557Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-05-01T00:26:36.482220057Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-01T00:26:36.482220057Z }","propolis_id":"c024bad3-d47c-45d2-898f-93d15935eabc"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.482272597Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.482319438Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.4850998Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:36.485790088Z","hostname":"buskin","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/392736c8-1ed9-4a79-90fc-44745da3ef39/debug/bundle/zone\", \"/pool/int/49112153-160e-48f4-abfa-c3642e83cda8/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/eaa88cfe-d08c-4ee9-8c11-845f26835ca2/crypt/debug\", \"/pool/ext/ad3dbf69-5e12-43e4-b601-7fce4e3f33b5/crypt/debug\", \"/pool/ext/4b494463-e11e-42a2-b533-5933dfea9e20/crypt/debug\", \"/pool/ext/0112b31e-4dc2-42d6-a774-9294ebc7e996/crypt/debug\", \"/pool/ext/2da812e0-f19d-4c88-af4f-92430c5507c5/crypt/debug\", \"/pool/ext/1bbd5d50-8020-48f2-ba68-03b3ab3e35af/crypt/debug\", \"/pool/ext/7b354c5b-27b9-467d-9c0f-816eaba8dbe6/crypt/debug\", \"/pool/ext/6464a1b3-39ff-4f6d-bb8d-28b483455b0c/crypt/debug\", \"/pool/ext/22b3b804-39bb-4b5d-9386-43e03fd38f0f/crypt/debug\"] }","zone_name":"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:37.120032694Z","hostname":"buskin","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc\", bundle_id: b16a09d0-5cb0-4b57-a457-b552cc76964c }, time_created: 2024-05-01T00:26:36.486051880Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_c024bad3-d47c-45d2-898f-93d15935eabc","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:37.120121406Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:38.317881466Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34055"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:38.317981148Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"9029878a-ef62-4a97-a7bb-48cd17553f71","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34055","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:38.877441456Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:38.902535811Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-01T00:26:36.482220057Z }, propolis_id: c024bad3-d47c-45d2-898f-93d15935eabc, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-01T00:26:38.902470260Z } }","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:38.973590967Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59780"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-05-01T00:26:38.974400216Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:38.974426647Z","hostname":"buskin","pid":1590,"uri":"/v2p/41785a54-3052-49f4-965a-a32a419f7c9f","method":"DELETE","req_id":"26a60a4c-6b84-40a7-914c-9d4fa856b59c","remote_addr":"[fd00:1122:3344:101::a]:59780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":712,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:39.247426564Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-01T00:26:36.482220057Z }, propolis_id: c024bad3-d47c-45d2-898f-93d15935eabc, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-01T00:26:38.902470260Z } }","instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:39.251704893Z","hostname":"buskin","pid":1590,"instance_id":"aa6f5f61-878c-4f0f-8c90-b19dbdb2d164","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:41.122738744Z","hostname":"buskin","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:41.122828794Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-01T00:26:41.156361725Z","hostname":"buskin","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999394s","next_cleanup":"Instant { tv_sec: 2543, tv_nsec: 3417788 }"}