[ Apr 3 22:19:12 Enabled. ] [ Apr 3 22:19:12 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Apr 3 22:19:12 Method "start" exited with status 0. ] [ Apr 3 22:19:12 Rereading configuration. ] [ Apr 3 22:19:12 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:12.294572804Z","hostname":"buskin","pid":1618,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:12.294753099Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:253"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:12.34793934Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:260","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:12.347986791Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:12.463808105Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:12.472539318Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:12.48051978Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:270","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.082569147Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.220384023Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:115"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.220442354Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:127"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.220518686Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:137"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.22067164Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:157","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.220693801Z","hostname":"buskin","pid":1618,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.220703901Z","hostname":"buskin","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.220761813Z","hostname":"buskin","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.223982243Z","hostname":"buskin","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:13.224013784Z","hostname":"buskin","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.224022154Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:171"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.224029444Z","hostname":"buskin","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.224074055Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.224107826Z","hostname":"buskin","pid":1618,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:261","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.22424065Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.22426318Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.224300632Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.236693277Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.236733487Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:312","id":"38c50e2b-3905-4ab7-a650-3ad6b1287295"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.337749739Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.536140581Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.536174182Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.536179222Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.536190793Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.536196463Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.536201153Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.5443644Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.544395201Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:312","id":"939cb7b8-5f2f-4f5c-821a-6bf92b625674"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.629237862Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827347866Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827394268Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827430849Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827443949Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827449939Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.82749143Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827511351Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827521561Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827561452Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827570763Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827586293Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827603973Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827609364Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827613804Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827633204Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827637494Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827642245Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827651645Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827655985Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827660825Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827679196Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827684586Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827688616Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827703906Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827722657Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827727877Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827731957Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827739237Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827759318Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827764348Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827768368Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827788039Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827809399Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827814879Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.82782083Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.82785262Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827858251Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:87"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827870021Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:89"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827874251Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827899312Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827905292Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:195"} {"msg":"No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827950573Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827970424Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827976984Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.827992024Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.828002204Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.828007215Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.828036276Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.828061376Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:215"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.837980722Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:709"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.838016983Z","hostname":"buskin","pid":1618,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999931608s","next_cleanup":"Instant { tv_sec: 1615, tv_nsec: 896857368 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.838069435Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3596"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.838079295Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.838133806Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.838147267Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.840550634Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.840629686Z","hostname":"buskin","pid":1618,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.840841312Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:997"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.840848992Z","hostname":"buskin","pid":1618,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.840897883Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.840940025Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.840955265Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.840960865Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1057"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.840992976Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841002757Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841018457Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841023577Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1098"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841027727Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841031427Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841056308Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841065708Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841432578Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841582173Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841594413Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1131"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841647694Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841655855Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841804569Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841937083Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.841994644Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 1ee9e643-97de-413d-8b33-615f41c711a3, rack_id: 6418b01e-3d3b-4688-85d2-17d265929a3a, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842122918Z","hostname":"buskin","pid":1618,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:33406","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 1ee9e643-97de-413d-8b33-615f41c711a3, rack_id: 6418b01e-3d3b-4688-85d2-17d265929a3a, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842319953Z","hostname":"buskin","pid":1618,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:356"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842330623Z","hostname":"buskin","pid":1618,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:371"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842634632Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842638872Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/server.rs:44"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842642522Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842668993Z","hostname":"buskin","pid":1618,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842673893Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842773286Z","hostname":"buskin","pid":1618,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842803676Z","hostname":"buskin","pid":1618,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842807867Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.842811207Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.845547653Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3827"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.853125614Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.862803143Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.873206603Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.883072607Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.891769899Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.900684708Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.909570185Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.919001077Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.927884045Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.927938136Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.927943096Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.927965327Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.927974837Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.927984178Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.94892802Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.966624543Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.985861488Z","hostname":"buskin","pid":1618,"sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:355"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.985912549Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.985919809Z","hostname":"buskin","pid":1618,"sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:367"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.985926499Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 74.82811ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:13.988679486Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"74835456-7ded-40b3-a5f8-e4c640ec0143\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:19:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"74835456-7ded-40b3-a5f8-e4c640ec0143\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_38c50e2b-3905-4ab7-a650-3ad6b1287295/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.000522786Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_38c50e2b-3905-4ab7-a650-3ad6b1287295/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.024282867Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_38c50e2b-3905-4ab7-a650-3ad6b1287295/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.024329698Z","hostname":"buskin","pid":1618,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.029244335Z","hostname":"buskin","pid":1618,"sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:382"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.029255165Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_38c50e2b-3905-4ab7-a650-3ad6b1287295/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.029259646Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.064118076Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 78.68691ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.064710102Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d3a2da66-2064-46e5-9bd0-d65b98c55bb2\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:19:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d3a2da66-2064-46e5-9bd0-d65b98c55bb2\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.094355248Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.144417231Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 201.645148ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.144948486Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e048e8e5-5932-4ac5-8ff7-8e4d1b00b881\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:19:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e048e8e5-5932-4ac5-8ff7-8e4d1b00b881\" }"} {"msg":"Mounting oxi_38c50e2b-3905-4ab7-a650-3ad6b1287295/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.311653616Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.323030332Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.323115875Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.323154186Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.323175476Z","hostname":"buskin","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.347132033Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 145.499706ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.34774744Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8d7740a9-d9b2-4fed-a329-f158d2526a39\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:19:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8d7740a9-d9b2-4fed-a329-f158d2526a39\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.494783773Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 147.396109ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.495255456Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9d5d1cdb-af7a-4f90-8e79-566ccded8cf9\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:19:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9d5d1cdb-af7a-4f90-8e79-566ccded8cf9\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.643184123Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 239.213796ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.643957845Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9724f807-e8ca-4bf6-98da-193dcdb9a08a\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:19:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9724f807-e8ca-4bf6-98da-193dcdb9a08a\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.88486754Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 156.051536ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:14.885642282Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1a1f9eea-823f-40c6-9014-89ab04633ccf\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:19:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1a1f9eea-823f-40c6-9014-89ab04633ccf\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.02677266Z","hostname":"buskin","pid":1618,"sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.042739674Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.077375338Z","hostname":"buskin","pid":1618,"component":"VmmReservoirManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:266"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.077415289Z","hostname":"buskin","pid":1618,"component":"VmmReservoirManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:274"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.077479821Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:991","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.077489941Z","hostname":"buskin","pid":1618,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.078569581Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.078607932Z","hostname":"buskin","pid":1618,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:426"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.078641913Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.078651653Z","hostname":"buskin","pid":1618,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.07888661Z","hostname":"buskin","pid":1618,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.079083416Z","hostname":"buskin","pid":1618,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:15.079089666Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.079095226Z","hostname":"buskin","pid":1618,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:261","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:15.079101266Z","hostname":"buskin","pid":1618,"sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"230.652165ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.079112046Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3596"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.079127707Z","hostname":"buskin","pid":1618,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:33406","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.28546005Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.285523911Z","hostname":"buskin","pid":1618,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.285632105Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.285715517Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.285722337Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.285739937Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.285755348Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:15.311297419Z","hostname":"buskin","pid":1618,"sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"685.625746ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.317664386Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47889"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.317829411Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.317909073Z","hostname":"buskin","pid":1618,"uri":"/inventory","method":"GET","req_id":"6df91ebc-c2cb-46a3-92a5-6f446320a371","remote_addr":"[fd00:1122:3344:101::1]:47889","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.354117771Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:57290"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.354196393Z","hostname":"buskin","pid":1618,"uri":"/sled-role","method":"GET","req_id":"6b9b0982-9bf6-43cf-8c7e-abf6b4f6d409","remote_addr":"[fd00:1122:3344:101::1]:57290","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":31,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.35551785Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.355545991Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.355845339Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.356122907Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:802"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 60971685-a36c-4f5c-9601-4f24fe9e6b1a, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: 7d954909-2e42-44b1-96d9-f536366f6758 }, OmicronPhysicalDiskConfig { id: 8ceab58c-1296-4cee-99a9-3edc6cda9420, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 0c6b0894-9c77-422a-8363-2705812a672c }, OmicronPhysicalDiskConfig { id: 6f9bdef3-12d3-45ca-a1cb-e7d1c3518880, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: af9b6346-5d28-46df-baea-640dad9b76c3 }, OmicronPhysicalDiskConfig { id: 634b2092-89fb-47fb-8789-2fa53e55f4bb, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e }, OmicronPhysicalDiskConfig { id: 024dc529-273d-481e-a961-c69d2315d5d9, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a }, OmicronPhysicalDiskConfig { id: bb568e2f-278a-44b1-9f4e-4fb26a5c9265, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4 }, OmicronPhysicalDiskConfig { id: 69dc89c3-a111-4e05-9cb2-7a24d8f4c29d, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8 }, OmicronPhysicalDiskConfig { id: baa01d53-698b-4890-8779-0595a5dcd58a, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: 70a16060-4587-476f-9772-16ea3a5a39aa }, OmicronPhysicalDiskConfig { id: c07aaaa5-d119-45c6-a06e-5772e5c4b5cf, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: 761d3524-c333-4332-9563-7d1ceb36709e }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.386878423Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:344"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.386946725Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:46916"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: 60971685-a36c-4f5c-9601-4f24fe9e6b1a, pool_id: 7d954909-2e42-44b1-96d9-f536366f6758 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: 8ceab58c-1296-4cee-99a9-3edc6cda9420, pool_id: 0c6b0894-9c77-422a-8363-2705812a672c }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: 6f9bdef3-12d3-45ca-a1cb-e7d1c3518880, pool_id: af9b6346-5d28-46df-baea-640dad9b76c3 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: 634b2092-89fb-47fb-8789-2fa53e55f4bb, pool_id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: 024dc529-273d-481e-a961-c69d2315d5d9, pool_id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: bb568e2f-278a-44b1-9f4e-4fb26a5c9265, pool_id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: 69dc89c3-a111-4e05-9cb2-7a24d8f4c29d, pool_id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: baa01d53-698b-4890-8779-0595a5dcd58a, pool_id: 70a16060-4587-476f-9772-16ea3a5a39aa }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: c07aaaa5-d119-45c6-a06e-5772e5c4b5cf, pool_id: 761d3524-c333-4332-9563-7d1ceb36709e }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.38715685Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.38716328Z","hostname":"buskin","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.387169011Z","hostname":"buskin","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.387174081Z","hostname":"buskin","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.387178581Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.387183351Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.387189371Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.387193981Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.496180065Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:15.496221426Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"7d954909-2e42-44b1-96d9-f536366f6758"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:16.178562187Z","hostname":"buskin","pid":1618,"sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"1.238479651s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:16.17899367Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:16.625207819Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:16.822227293Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:16.822362297Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:16.822373587Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:16.822396187Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:16.822409628Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:16.822524871Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:16.822533611Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:17.296053281Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:17.296462802Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:17.506890889Z","hostname":"buskin","pid":1618,"sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"1.419221256s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:17.509539413Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"} {"msg":"Automatically destroying dataset: oxp_7d954909-2e42-44b1-96d9-f536366f6758/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:17.736995474Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:17.919821362Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:18.559212348Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 55.399324ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:18.76767098Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:19.164632759Z","hostname":"buskin","pid":1618,"sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"4.323909911s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:19.164708571Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 60.364809ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:19.165095562Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:19.865562178Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 233.709658ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:19.865700082Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 201.432853ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:20.684717667Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_7d954909-2e42-44b1-96d9-f536366f6758","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:21.107458313Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:21.107507314Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 799.078609ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:21.319505545Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:21.322483398Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:21.327457807Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:21.327471087Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:21.327514558Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:21.327541979Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:21.327548649Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:21.526599059Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:21.52664202Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"0c6b0894-9c77-422a-8363-2705812a672c"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:22.621625437Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:22.834940055Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:22.835009287Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:22.835021257Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:22.835032927Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:22.835040217Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:22.835170631Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:22.835181181Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:23.348875819Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:23.349009533Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:23.805418076Z","hostname":"buskin","pid":1618,"sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1190","retry_after":"10.163736533s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:24.295797184Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:24.763738299Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_0c6b0894-9c77-422a-8363-2705812a672c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:24.763913463Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.413218595Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2294"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.61779943Z","hostname":"buskin","pid":1618,"component":"VmmReservoirManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:278"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:25.619712723Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_0c6b0894-9c77-422a-8363-2705812a672c","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.730511117Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.738397006Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.738443168Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.738449398Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.738456958Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.738461548Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.748547819Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.74858907Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"af9b6346-5d28-46df-baea-640dad9b76c3"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.841845586Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.849579381Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.849614742Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.849622302Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.849647803Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.849658023Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.849770856Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.849780306Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:25.853442568Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.88582903Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"} {"msg":"Automatically destroying dataset: oxp_af9b6346-5d28-46df-baea-640dad9b76c3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:25.893905365Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_af9b6346-5d28-46df-baea-640dad9b76c3","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.019230153Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.027167123Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.027217895Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.027229535Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.027237676Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.027249126Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.038015756Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.038053286Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"ae4e2e29-ff41-46eb-978b-335ffe5cd37e"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.132747152Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.141001262Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.141041303Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.141049223Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.141060834Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.141071234Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.141226488Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.141233809Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.177127968Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"} {"msg":"Automatically destroying dataset: oxp_ae4e2e29-ff41-46eb-978b-335ffe5cd37e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.1854905Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_ae4e2e29-ff41-46eb-978b-335ffe5cd37e","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.311721764Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.319517541Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.319551502Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.319571822Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.319580593Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.319587903Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.329614472Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.329645642Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"4eb310e9-0d98-4a07-821c-54f35a32dc4a"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.421823648Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.429759339Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.42978799Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.42979564Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.429822921Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.429834981Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.430043307Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.430054597Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.465889125Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"} {"msg":"Automatically destroying dataset: oxp_4eb310e9-0d98-4a07-821c-54f35a32dc4a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.473859976Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_4eb310e9-0d98-4a07-821c-54f35a32dc4a","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.597518329Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.605769138Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.605808029Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.605861201Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.605871691Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.605879441Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.61552019Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.61555048Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"494a3a1c-f913-4e4b-8eec-202ec6c5bfd4"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.708737344Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.716544461Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.716597683Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.716613643Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.716657545Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.716672755Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.716818769Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.716827249Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.753085908Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"} {"msg":"Automatically destroying dataset: oxp_494a3a1c-f913-4e4b-8eec-202ec6c5bfd4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.761021429Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:26.858470872Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_494a3a1c-f913-4e4b-8eec-202ec6c5bfd4","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.883667693Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.891441289Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.89147829Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.891502651Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.891528352Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.891536832Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.900977145Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.901005676Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"00b9d2d0-59a4-4682-aa18-072c6202c0c8"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:26.994680482Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.002829639Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.00286691Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.002883371Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.002946232Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.002958173Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.003098177Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.003107657Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.038240684Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"} {"msg":"Automatically destroying dataset: oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.046371191Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.169440366Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.177280305Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.177315426Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.177323046Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.177328036Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.177334676Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.187305364Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.187339874Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"70a16060-4587-476f-9772-16ea3a5a39aa"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.282599756Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.290080054Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.290114855Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.290124615Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.290169677Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.290179987Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.290349792Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.290359172Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:27.324894853Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.326290582Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"} {"msg":"Automatically destroying dataset: oxp_70a16060-4587-476f-9772-16ea3a5a39aa/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.334354666Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_70a16060-4587-476f-9772-16ea3a5a39aa","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.457561806Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.465638231Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.465672992Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.465681372Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.465690062Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.465738983Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.475809904Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.475838964Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"761d3524-c333-4332-9563-7d1ceb36709e"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.57412878Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.584037806Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.584088828Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.584102658Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.584113918Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.584125118Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.584248612Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.584257932Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.624427Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"} {"msg":"Automatically destroying dataset: oxp_761d3524-c333-4332-9563-7d1ceb36709e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.632449223Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_761d3524-c333-4332-9563-7d1ceb36709e","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.751902608Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.758910213Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.758980105Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.758995776Z","hostname":"buskin","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.75913806Z","hostname":"buskin","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 38c50e2b-3905-4ab7-a650-3ad6b1287295, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 939cb7b8-5f2f-4f5c-821a-6bf92b625674, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.759267293Z","hostname":"buskin","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.759656994Z","hostname":"buskin","pid":1618,"uri":"/omicron-physical-disks","method":"PUT","req_id":"ab7b66d3-e652-4720-80f6-8ce157fa6938","remote_addr":"[fd00:1122:3344:101::1]:46916","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12372644,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.790589655Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.790789391Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43473"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.791028267Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.791038177Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.836662718Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.836692298Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:27.861020145Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.885668891Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.885705643Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.930737616Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.930783017Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.930788967Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.930839329Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.971899951Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.971943403Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:27.971950323Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:28.019949029Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:28.023670392Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:28.027368065Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:28.039693938Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:28.044745739Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:28.048523814Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:28.864336691Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:29.322974336Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/1ee9e643-97de-413d-8b33-615f41c711a3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-03T22:19:29.866426082Z","hostname":"buskin","pid":1618,"component":"ProbeManager","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:30.42362561Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:30.423717413Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:30.424060953Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:30.906537202Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:31.387395765Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:31.387494697Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640\") failed: Property not found. retry in 43.569159ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.395311915Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:31.395355966Z","hostname":"buskin","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:31.395452499Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:31.400115709Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640\") failed: Property not found. retry in 82.662707ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.449964046Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca\") failed: Property not found. retry in 61.178837ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.521321862Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e\") failed: Property not found. retry in 72.516622ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.528811311Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640\") failed: Property not found. retry in 295.895179ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.539503199Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca\") failed: Property not found. retry in 88.46188ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.638884825Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e\") failed: Property not found. retry in 81.924751ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.696617372Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca\") failed: Property not found. retry in 215.930069ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.744843044Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e\") failed: Property not found. retry in 176.077063ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.835631341Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640\") failed: Property not found. retry in 444.834572ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.858110216Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca\") failed: Property not found. retry in 410.230104ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:31.989945035Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e\") failed: Property not found. retry in 390.927711ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:32.023825328Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640\") failed: Property not found. retry in 1.01788785s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:32.312269597Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca\") failed: Property not found. retry in 1.157318256s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:32.408806324Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e\") failed: Property not found. retry in 1.038204673s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:32.422258298Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640\") failed: Property not found. retry in 768.634957ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:33.34064777Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e\") failed: Property not found. retry in 507.757197ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:33.471591654Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca\") failed: Property not found. retry in 1.189869056s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:33.576092003Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:33.641308648Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2311"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:33.641325829Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e\") failed: Property not found. retry in 660.040542ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:33.988935313Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640\") failed: Property not found. retry in 1.164775827s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:34.117358098Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fedf:8a3e","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:34.186610465Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2318"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:34.1925138Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2379"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:34.313923479Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2564"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:34.560543243Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2728"} {"msg":"wait for service Some(\"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e\") failed: Property not found. retry in 1.326623155s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:34.658473689Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:34.756063275Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2443"} {"msg":"wait for service Some(\"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca\") failed: Property not found. retry in 691.374532ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:34.776556025Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:35.055031536Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2478"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:35.260954388Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766"} {"msg":"wait for service Some(\"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640\") failed: Property not found. retry in 658.794292ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:35.292108875Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:35.446045759Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2793"} {"msg":"wait for service Some(\"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca\") failed: Property not found. retry in 1.236379606s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:35.479979464Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:35.668518202Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2672"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:35.858648874Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2472"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:35.868837227Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3851","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:35.868865958Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3867"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:35.868870328Z","hostname":"buskin","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640\") failed: Property not found. retry in 893.306269ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:35.961733533Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e\") failed: Property not found. retry in 909.619501ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:35.994399501Z","hostname":"buskin","pid":1618,"zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.423077023Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3875"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.429935694Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3931"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.583159828Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3993"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.706270545Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3974","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.73518716Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:4016"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.850719666Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.925339483Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_internal_dns_82d8d423-6846-428f-8eb7-7d87943ccbca"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.925419055Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_internal_dns_711ccb54-227e-476e-9bc7-c738bae2623e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.925425995Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_internal_dns_ea42831f-f0e1-4084-840f-aa951f986640"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.925466506Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.925857247Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.926147765Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"be30061a-13c2-4fd5-901f-9d8d52d6ffbf","remote_addr":"[fd00:1122:3344:101::1]:43473","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9135283,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.92776156Z","hostname":"buskin","pid":1618,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:545"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:36.979554841Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.011704986Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.043145491Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:589"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.043231964Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:627"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.043686196Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:630","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.043701677Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.11783229Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117914742Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117924803Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117934003Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117942693Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117949603Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117956213Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117962924Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117968234Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117979644Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117986414Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.117991334Z","hostname":"buskin","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"failed to write DNS configuration (will retry in 144.219177ms)","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:37.25435771Z","hostname":"buskin","pid":1618,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:565","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.416459262Z","hostname":"buskin","pid":1618,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:545"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.461616979Z","hostname":"buskin","pid":1618,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:545"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.507143076Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:581"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.507154026Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.538046206Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.538745755Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.538751676Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.567057393Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.567215858Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58502"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.567441784Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.567447524Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.589480368Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.589519418Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.609525655Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.609558956Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.627588588Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.62764448Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.627813584Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.627930088Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.639696645Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.671572692Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.672288032Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.672310653Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.716174014Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 219, 117]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.716223025Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1285","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.716790491Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.716800681Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.717072669Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.718974082Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.743508465Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:37.762066901Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:38.888913315Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:38.889125501Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5\") failed: Property not found. retry in 47.933486ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:39.021744832Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5\") failed: Property not found. retry in 65.260398ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:39.077645637Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5\") failed: Property not found. retry in 170.551358ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:39.157250093Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5\") failed: Property not found. retry in 291.99534ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:39.338469547Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5\") failed: Property not found. retry in 988.411442ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:39.639670691Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:40.311448148Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61080"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:40.311615693Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"9b383a6b-5b90-49b0-8c81-448457580973","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61080","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":82,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5\") failed: Property not found. retry in 824.069415ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:40.638176062Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5\") failed: Property not found. retry in 530.070282ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:41.47131354Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5\") failed: Property not found. retry in 1.065131445s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:42.01239236Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5\") failed: Property not found. retry in 733.522046ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:43.088193793Z","hostname":"buskin","pid":1618,"zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:43.843139296Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:43.843183087Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:43.843516346Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:43.843775453Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"48d4a961-9db1-4820-887b-62da02488b84","remote_addr":"[fd00:1122:3344:101::1]:58502","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6276507,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:43.845623845Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:622"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:43.874575471Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:43.874718335Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45639"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:43.885117084Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:43.885246108Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"0e661e4d-320f-4e5a-a1dd-47476b94add8","remote_addr":"[fd00:1122:3344:101::1]:45639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":10403,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:43.888148759Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:44.017410506Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:44.017501398Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:55364"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.028009523Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.028218529Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"58abd866-b85a-4f4d-ad65-8f0921c407f8","remote_addr":"[fd00:1122:3344:101::1]:55364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4010610,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:48.031932972Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.176382802Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.176509056Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43888"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.185782094Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.185905548Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"38551ec0-6345-495e-b03e-542d7787b720","remote_addr":"[fd00:1122:3344:101::1]:43888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9256,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:48.187342908Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.4119947Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.412132474Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35752"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4933dd3d-baa4-43a7-b488-a6e5fdad89d5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.420390474Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.420514168Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"8f4133d3-113e-4943-aac8-4bd6a8c800ca","remote_addr":"[fd00:1122:3344:101::1]:35752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8251,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:48.422204885Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.564005031Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:48.564096254Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:50.572194625Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"2cada7f3-dc19-4275-9dc0-d81764e83c6c","remote_addr":"[fd00:1122:3344:101::1]:51735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2007981,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:50.574393217Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:50.574401757Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:50.812220216Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:50.81235521Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:50.820999741Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"c8256890-d0e6-438e-9964-ffd42f1666cc","remote_addr":"[fd00:1122:3344:101::1]:54801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8513,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:50.823731066Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:50.823738167Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.100268103Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.100429938Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51400"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.108923984Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"aaf82ed8-8b78-4cf3-805b-d97aab01c903","remote_addr":"[fd00:1122:3344:101::1]:51400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8456,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.111438304Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:51.111447804Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.499165456Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.499297199Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35283"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.507995011Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"10d184a1-c51a-4389-bd21-86b484c3c485","remote_addr":"[fd00:1122:3344:101::1]:35283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8625,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.5097272Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:51.5097325Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.914137786Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.914246319Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58489"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.922731525Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"db58f926-fccb-4d7c-a279-c374ed096d01","remote_addr":"[fd00:1122:3344:101::1]:58489","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8366,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:51.924501314Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:51.924510575Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:52.265138665Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:52.265246018Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63457"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:52.273609521Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"7dd78565-0f65-44b9-832e-4a1c40f5e13d","remote_addr":"[fd00:1122:3344:101::1]:63457","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8226,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:52.275785681Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:52.275792781Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:52.972244946Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:52.972356359Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:46614"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:52.980968488Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"747dc529-024f-4904-badb-0372ed8be49a","remote_addr":"[fd00:1122:3344:101::1]:46614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8482,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1712182787.540047, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:52.982683946Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:52.982689407Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.174368473Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.174543608Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54830"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.178039816Z","hostname":"buskin","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.19617073Z","hostname":"buskin","pid":1618,"uri":"/timesync","method":"GET","req_id":"e3d2c785-edfd-483c-ab1e-6ebdcf650f2e","remote_addr":"[fd00:1122:3344:101::1]:54830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":21540,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380417, ip_addr: 162.159.200.1, stratum: 4, ref_time: 1712182792.50853, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.198337971Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.198349351Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1210"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.227268166Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.227448901Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61922"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.227649767Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.227661957Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.246015938Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.246058409Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.264899033Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.264952815Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.2820512Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.282110762Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.33486596Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.334915132Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.392111204Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.392156425Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.44549837Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.445542581Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.497140837Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.497199359Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.547470828Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.547525159Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.547779866Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.54791316Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.62729994Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.627370302Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.627377872Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.627383172Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.627392042Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.719377373Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.723273501Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.727138889Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.73149155Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.735983105Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.74049415Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.744680047Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.749051908Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.753451261Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:53.757851164Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.40435641Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.404507164Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.407761705Z","hostname":"buskin","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Profile for oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.410893982Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.411038016Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.418892654Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.419018378Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.42843296Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.428554883Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.450530565Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:19:57.450674999Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 64.341634ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:57.563322175Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 55.74056ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:57.570976978Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 27.462502ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:57.577723115Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 29.025169ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:57.585150822Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 26.837005ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:57.596622312Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 66.841369ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:57.612957286Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 68.536388ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:57.623114169Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 82.919355ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:57.745891347Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 124.278524ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:57.923437218Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 82.150206ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:58.022835795Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 178.743434ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:58.249413322Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 188.609569ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:58.521530776Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 108.572301ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:58.572565906Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 217.347288ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:58.710480795Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 101.994445ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:58.789244837Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 426.674006ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:58.966171112Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 298.749914ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:59.114605574Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 347.495999ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:59.218917357Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 337.819567ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:59.408962507Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 495.633027ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:19:59.652932277Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 1.171213993s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:00.433639728Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 469.846174ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:00.506707392Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 1.059205939s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:00.520760783Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 834.93827ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:00.529066154Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 494.919708ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:00.536321676Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 952.391033ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:00.988072411Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 913.986362ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:01.042670091Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 661.042749ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:01.374340852Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 506.258606ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:01.591035794Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 1.2129297s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:01.614072495Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 746.638021ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:01.950365096Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 1.142919381s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:01.967207504Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 1.169279627s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:02.044518596Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 813.106494ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:02.10535803Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 885.509721ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:02.70714567Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 1.390544729s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:02.836629864Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 1.421226778s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:02.928414049Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 886.820605ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:03.119437476Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 1.129895135s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:03.223611466Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 853.875675ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:03.603986483Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 724.868553ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:04.016897386Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 745.313633ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:04.239456631Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 799.527859ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:04.362226988Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 606.876483ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:04.373122321Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 667.985464ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:04.470554213Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 842.261845ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:04.753503449Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 566.419941ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:04.98921659Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 778.64542ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:04.998273392Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 720.361971ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:05.148870054Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 511.117183ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:05.171211466Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 546.696342ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:05.566758656Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 801.009013ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:05.604705152Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 1.166790533s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:05.690602663Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 1.404308789s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:05.785607417Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 1.217974075s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:05.877548016Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 1.412488559s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:06.122183245Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 1.451043598s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:06.415713266Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 691.623952ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:06.866861353Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d\") failed: Property not found. retry in 1.47380394s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:07.106274057Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0\") failed: Property not found. retry in 1.338805725s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:07.199580014Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac\") failed: Property not found. retry in 837.439996ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:07.544182416Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000\") failed: Property not found. retry in 989.067038ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:07.568235105Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 836.542652ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:07.878489701Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 980.827202ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:08.724464408Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 1.383012018s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:09.716519222Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:09.747285088Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34765"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:09.747437563Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"a40d465f-7506-4eca-9648-fc59b70a7205","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34765","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":69,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 957.526483ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:11.112005105Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 502.597775ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:12.133670082Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 515.220152ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:12.648020228Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 716.143635ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:13.17837379Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 982.407355ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:13.904507002Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 1.293010308s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:14.896142924Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 743.316009ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:16.199405969Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 1.163034171s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:16.951852493Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5\") failed: Property not found. retry in 1.176259619s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:18.127115366Z","hostname":"buskin","pid":1618,"zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.324232737Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_cockroachdb_70323a7d-e09e-4fb2-b8a6-3cb5eaff9000"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.324280358Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_cockroachdb_e35cba43-078c-4794-99ca-4255942899ac"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.324286978Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.324291538Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_cockroachdb_760e9943-67c9-4971-982a-df71813585c0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.324296759Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_cockroachdb_cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.3243323Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.430605418Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.490843114Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"28fa9715-c05e-4731-b57b-9304c1ac1bb7","remote_addr":"[fd00:1122:3344:101::1]:61922","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":26166483,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.524358247Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38484"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.557736966Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"d2eff7e6-9008-4714-83ce-28f752365d55","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":33191,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:19.557975563Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"76.990711ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2eff7e6-9008-4714-83ce-28f752365d55\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2eff7e6-9008-4714-83ce-28f752365d55\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.662972036Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"43427e4b-7ec2-4231-afa2-b50c8c73959b","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26667,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:19.663160481Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"136.497358ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"43427e4b-7ec2-4231-afa2-b50c8c73959b\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"43427e4b-7ec2-4231-afa2-b50c8c73959b\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.824642376Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"fe40fe15-8708-4183-8fd7-a88809ab44b1","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":24294,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:19.82481322Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"113.0573ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe40fe15-8708-4183-8fd7-a88809ab44b1\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fe40fe15-8708-4183-8fd7-a88809ab44b1\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:19.966111203Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"105e237d-0be1-4c8c-834c-aa2d34b3c5f1","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27086,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:19.966281678Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"115.562004ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"105e237d-0be1-4c8c-834c-aa2d34b3c5f1\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"105e237d-0be1-4c8c-834c-aa2d34b3c5f1\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:20.110914994Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"3ad47092-85f5-45a3-9e88-e60165906be7","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27466,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:20.111055148Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"169.771217ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3ad47092-85f5-45a3-9e88-e60165906be7\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3ad47092-85f5-45a3-9e88-e60165906be7\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:20.30800389Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"f96e824f-0844-480c-b422-76fb380617a2","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":25749,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:20.308173015Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"297.120078ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f96e824f-0844-480c-b422-76fb380617a2\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f96e824f-0844-480c-b422-76fb380617a2\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:20.632960644Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"22a7cfda-438c-4b13-a640-18a150fb413d","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26595,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:20.633111379Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"441.328299ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"22a7cfda-438c-4b13-a640-18a150fb413d\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"22a7cfda-438c-4b13-a640-18a150fb413d\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:21.103560434Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"1db6d45d-eafb-492e-b1ae-f3ab8bf94a33","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28151,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:21.103762619Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"422.754449ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1db6d45d-eafb-492e-b1ae-f3ab8bf94a33\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1db6d45d-eafb-492e-b1ae-f3ab8bf94a33\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:21.556290915Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"be675faf-6ca6-486c-9457-eab0e70f0430","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":29201,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:21.55647888Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"640.134791ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be675faf-6ca6-486c-9457-eab0e70f0430\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"be675faf-6ca6-486c-9457-eab0e70f0430\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:22.225446471Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"f30b01c4-8c0d-48ee-9840-186346bb98af","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27617,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:22.225616626Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"290.216482ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f30b01c4-8c0d-48ee-9840-186346bb98af\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f30b01c4-8c0d-48ee-9840-186346bb98af\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:22.544009557Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"16a02198-afbc-4c96-861e-7bd50487130f","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4712897e-ce28-4ed0-ae00-c454b60a50d5 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26937,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:22.544212423Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:952","retry_after":"766.791925ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"16a02198-afbc-4c96-861e-7bd50487130f\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"16a02198-afbc-4c96-861e-7bd50487130f\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:23.341713067Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3416"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:29.13468511Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3434"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.157033283Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3457"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.157148345Z","hostname":"buskin","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"5414fae1-6644-4125-a58d-24e2f3a5f3d7","remote_addr":"[fd00:1122:3344:101::1]:38484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":11812245,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.191089292Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.191356759Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53174"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.191663948Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.191669142Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.215037204Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.215082844Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.235892429Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.235939043Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.256010956Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.25613551Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.274719677Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.274763159Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.293463952Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.293503975Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.313325602Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.313370303Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.33287709Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.33291596Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.350563441Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.350616251Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.450203399Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.450252966Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.55314912Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.553210425Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.664588013Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.664639839Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.763272638Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.763318305Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.823201652Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.823253732Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.89959038Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.89964036Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.98717872Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:35.9872281Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.05913032Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.059248616Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.120937463Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.120983027Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.196818493Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.196869336Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.260377484Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.26042062Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.340710667Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.340773977Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.341061191Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.341224827Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438825125Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438868354Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438876661Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438882576Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438889683Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438913805Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438921024Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.43892649Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.43893353Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.4389391Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438944688Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438950322Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438955572Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438960747Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438965978Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438971042Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438975898Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438982629Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.438988057Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.439109739Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"6bba5f07-5399-4b2b-8300-fd03ade49d0d","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized)]","latency_us":1244399,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:36.43944619Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"90.150003ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6bba5f07-5399-4b2b-8300-fd03ade49d0d\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6bba5f07-5399-4b2b-8300-fd03ade49d0d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.530825431Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.531319141Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.531326089Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.551833276Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.55187252Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.57324045Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.573282291Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.591093165Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.591133638Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.611313032Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.611347646Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.629227085Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.629264023Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.646916533Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.6469547Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.664579127Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.664618016Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.684507461Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.684549548Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.702840572Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.702955964Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.72285338Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.722893301Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.742897874Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.742937194Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.761834432Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.761871024Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.780811174Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.780857787Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.803084403Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.803124876Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.82250719Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.822543125Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.841813722Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.841855573Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.861576585Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.861611554Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.881672724Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.881728732Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.900358509Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.900467481Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.920741105Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.920783303Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.920971005Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:36.921099437Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.01583609Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015882928Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.0158895Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015895032Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015901839Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015907576Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015922632Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015929683Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015935327Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015940887Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015946343Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015953412Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015959102Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015964334Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015974731Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015980347Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015985925Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015992797Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.015999107Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.016177518Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"ee07c6d8-c3ec-4364-a71e-ab7a12d71ee2","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized)]","latency_us":483732,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.016406022Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"161.443097ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ee07c6d8-c3ec-4364-a71e-ab7a12d71ee2\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ee07c6d8-c3ec-4364-a71e-ab7a12d71ee2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.180062727Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.181207754Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.181215947Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.203600889Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.203647531Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.226379994Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.226427649Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.24805379Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.248100095Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.267067297Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.267118148Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.285261917Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.285307951Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.304092915Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.304140616Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.32501455Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.325060912Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.344829296Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.34487381Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.365017023Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.365059062Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.38427942Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.384323005Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.40333581Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.403381627Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.422608993Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.422656281Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.440928242Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.440976955Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.458765831Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.458810016Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.477736479Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.477776399Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.496343298Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.496383921Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.518205519Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.51824693Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.537456903Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.537504501Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.557715304Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.557759808Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.576333953Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.576384429Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.576590835Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.576730293Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672893787Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672928194Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.67293426Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672940166Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672944253Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672949391Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672953395Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672973101Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672977526Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672982279Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672985383Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672989292Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672993165Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.672997102Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.673001762Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.67300568Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.673010508Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.67301454Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.673019405Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.67314059Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"5377c6a5-8700-4b1c-8dc1-9049c2fd2b72","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized)]","latency_us":491452,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:37.673362447Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"121.192199ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5377c6a5-8700-4b1c-8dc1-9049c2fd2b72\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5377c6a5-8700-4b1c-8dc1-9049c2fd2b72\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.796110795Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.796611649Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.796619806Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.819134869Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.819174741Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.840876689Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.840979693Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.861033958Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.861068984Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.880960567Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.880994909Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.90158739Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.901621506Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.920829307Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.920861165Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.940093106Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.940124167Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.958302371Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.95833721Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.978653982Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.978701205Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.999242518Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:37.999277722Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.021947966Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.021985504Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.04657826Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.046618893Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.067891814Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.067923061Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.088053308Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.088094841Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.107426449Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.107469415Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.12750661Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.127542227Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.147656497Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.147694204Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.173669816Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.173708376Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.194428782Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.19446108Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.216175253Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.21621267Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.2164379Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.216620428Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316502069Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316555256Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316561499Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316568634Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316574512Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316581384Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316586963Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.3165924Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316599338Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316612828Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316619551Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316625016Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316630201Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316635432Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316640532Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316647461Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316653011Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316658429Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.316665649Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.316804712Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"1573eeb3-1a29-4968-a102-e093b80b2164","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized)]","latency_us":518992,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.317040247Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"88.06165ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1573eeb3-1a29-4968-a102-e093b80b2164\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1573eeb3-1a29-4968-a102-e093b80b2164\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.406536935Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.407021054Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.407037705Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.431226117Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.431279557Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.453675219Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.453731986Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.475866196Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.475914676Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.498147386Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.498195424Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.519199713Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.519244602Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.541614719Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.541659027Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.562566839Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.562609899Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.583736612Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.58378031Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.606989094Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.607035754Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.626367773Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.626410692Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.648848678Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.648894843Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.671155556Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.671213439Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.692564689Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.692622638Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.718903994Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.71896572Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.743975461Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.7440205Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.767590071Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.767644082Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.793104319Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.793154899Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.818753931Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.818817392Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.844475316Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.844539161Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.869942165Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.870012217Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.870231054Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.870371281Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985505092Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985552756Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985558831Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985564109Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985568281Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.9855746Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985578959Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985583047Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985587313Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985592385Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985615027Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985618421Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.98562488Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985628939Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985632877Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985637649Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985641689Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985647671Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.985651684Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:38.985782571Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"cba19c94-b9d7-421f-abf7-9cb6b796fb85","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized)]","latency_us":577443,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:38.986100705Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"260.919349ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cba19c94-b9d7-421f-abf7-9cb6b796fb85\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cba19c94-b9d7-421f-abf7-9cb6b796fb85\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.249150251Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.249716256Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.249723269Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.272933522Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.272993393Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.295286655Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.295328984Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.315422663Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.315463576Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.33605569Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.336098637Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.355067438Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.355114615Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.37464256Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.374684364Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.39418114Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.394229309Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.415440851Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.415477265Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.435852828Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.435892064Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.455390871Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.455444825Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.474389396Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.47443976Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.494830646Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.494874842Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.515102034Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.515143838Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.533871251Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.533906736Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.552250334Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.552298073Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.572351764Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.572392368Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.590786188Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.590825677Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.610272172Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.610308483Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.632005635Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.632042527Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.651986995Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.652042319Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.652226871Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.652346602Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.748944044Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749067549Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749075236Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749081461Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749087584Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749093528Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749098515Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749104497Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749109719Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749115898Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749121054Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749135519Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749141707Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749147567Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749152527Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749158424Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749163074Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749167836Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749173639Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.749283423Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"d521b06c-6b9c-4bc6-bc25-81fe7897a223","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized)]","latency_us":498457,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:39.749930579Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"190.331832ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d521b06c-6b9c-4bc6-bc25-81fe7897a223\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d521b06c-6b9c-4bc6-bc25-81fe7897a223\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.8223441Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55661"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.822429021Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"8b559960-9817-47c7-b65e-7365ba36c980","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55661","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":40,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.942447997Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.943358224Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.943368705Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.964678546Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.964719301Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.984865344Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:39.984909164Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.003866623Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.003900326Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.02447326Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.024506552Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.043136231Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.043170648Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.062978466Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.063013174Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.085412748Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.085455049Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.105398203Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.105434306Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.123818921Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.123858766Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.143192856Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.143224404Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.161466789Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.161499687Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.180369672Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.180402992Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.200281549Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.200319696Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.219134506Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.219177567Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.238214529Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.238312334Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.258581929Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.258612605Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.277304142Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.277334978Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.295754513Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.295791395Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.316651397Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.316684717Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.336461077Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.336584182Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.336797573Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.336933205Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432226122Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432279635Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432287773Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432293427Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432300168Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432306093Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432311652Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.43231873Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.43232293Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432345009Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432351918Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432358679Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432363994Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.43236916Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.43237576Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432380907Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432386157Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432393892Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432402977Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.432538337Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"f51d49ff-dfb0-4048-a9a2-ba3a5834d363","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized)]","latency_us":488479,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:40.432838662Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"214.23252ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f51d49ff-dfb0-4048-a9a2-ba3a5834d363\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f51d49ff-dfb0-4048-a9a2-ba3a5834d363\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.648605887Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.649065247Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.649080903Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.671833551Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.67188339Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.693104889Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.693160081Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.714152222Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.714195405Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.733424022Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.733470702Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.751669236Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.751721072Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.771835904Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.771883222Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.790691898Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.790802709Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.809440134Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.809489983Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.830244722Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.830285288Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.851296589Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.851342358Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.872913695Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.873019803Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.893173592Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.893221875Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.914524803Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.914567845Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.9342137Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.934258448Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.953980833Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.954026266Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.97432334Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.974372111Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.99423784Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:40.994345435Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.016547507Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.016578558Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.037117574Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.037168172Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.056399643Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.056504792Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.056718285Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.056840436Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156177665Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.15622756Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.15623506Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.15623897Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156245898Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156251693Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156257018Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156262324Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156267518Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156272721Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156290956Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.15629674Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156301972Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156307212Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156313428Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156318651Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156323638Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156330154Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156335019Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.156436638Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"12118638-ae3a-49e1-ac6e-1da7ea919cc4","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized)]","latency_us":506219,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:41.156654387Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"354.975756ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"12118638-ae3a-49e1-ac6e-1da7ea919cc4\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"12118638-ae3a-49e1-ac6e-1da7ea919cc4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.514214569Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.515202022Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.515209784Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.539534481Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.539584067Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.560274786Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.560324971Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.5803771Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.580422748Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.599092646Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.599138219Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.618558259Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.618615194Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.637423521Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.637470294Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.65729392Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.657344753Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.676185238Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.67622813Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.69570983Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.695754635Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.716315605Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.716375578Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.735938113Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.735981605Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.756584711Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.756631597Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.776504831Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.776554061Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.795606157Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.795657448Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.814736097Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.81478228Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.833634094Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.833677249Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.853638216Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.853683621Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.873006328Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.8730516Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.89414228Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.894186953Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.914192045Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.914243909Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.914413086Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:41.914525992Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012098179Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012151216Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012157891Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012163309Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012170089Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012175554Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012181217Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012186776Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012192129Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012199432Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.01220486Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.01222678Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012233699Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012237647Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.0122427Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012249272Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012254728Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.0122595Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012266287Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.012456146Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"39a0c8d4-0f14-4e64-a0fe-79c92d759ef3","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized)]","latency_us":496574,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.012726885Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"285.090275ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"39a0c8d4-0f14-4e64-a0fe-79c92d759ef3\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"39a0c8d4-0f14-4e64-a0fe-79c92d759ef3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.299779085Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.300255976Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.300264874Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.323101734Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.323156964Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.344700761Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.344751246Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.367295017Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.367342831Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.386902553Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.387013239Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.40788719Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.407931694Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.429405836Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.42945469Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.449726511Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.449772881Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.467911824Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.467956676Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.488844261Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.488883478Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.506833053Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.506872365Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.527501615Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.527545163Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.548676751Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.548721405Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.568865572Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.568904526Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.587681159Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.587725635Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.606454331Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.606507067Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.626731185Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.626771386Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.646154878Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.646198154Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.664975189Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.665031348Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.683685368Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.683789657Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.70308247Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.703152156Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.703356293Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.703483197Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797908881Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.79795048Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797956621Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797960764Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797964655Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797968855Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797972783Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797976842Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797979937Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797983781Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797987962Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.797991825Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.798005372Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.798010481Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.798014551Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.798019172Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.798023194Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.798094035Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.798097888Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:42.798214628Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"c4074261-2e9e-4c02-bde9-829cc1c88135","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized)]","latency_us":496814,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:42.798438294Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"357.16682ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c4074261-2e9e-4c02-bde9-829cc1c88135\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c4074261-2e9e-4c02-bde9-829cc1c88135\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.157802659Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.158334943Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.158344243Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.182325961Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.182369659Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.203694511Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.203733662Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.224118087Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.224150582Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.244125129Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.244167824Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.263463055Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.263493609Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.283153818Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.283206881Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.303260953Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.303290569Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.322576165Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.32260594Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.342106056Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.342139948Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.361488063Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.361586042Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.380710043Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.380741226Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.400971457Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.401008058Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.420223737Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.420256344Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.440009598Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.440042993Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.460705391Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.460739067Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.480750847Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.480786192Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.499281839Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.499315965Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.51871394Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.518747372Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.539629276Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.539662464Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.559758611Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.559794837Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.560002442Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.560190157Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658608793Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.65865957Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658666367Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658673342Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658678686Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658684284Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658689834Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658695552Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.65870309Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658710252Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658715878Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658721363Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.65872681Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658733597Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658754673Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658760345Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658765585Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.658770573Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.65877706Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:43.658925762Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"4c9eb86c-c76e-4c24-9a73-1cfc0954c65f","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized)]","latency_us":499463,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:43.659148808Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"794.316093ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4c9eb86c-c76e-4c24-9a73-1cfc0954c65f\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4c9eb86c-c76e-4c24-9a73-1cfc0954c65f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.456668768Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.457114758Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.457123815Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.4805812Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.480629276Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.502180197Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.502246996Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.521685684Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.521739667Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.54238011Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.542428121Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.561157862Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.561202798Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.57909198Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.579144875Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.599527663Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.59957469Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.617778533Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.617830322Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.636900164Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.637009254Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.655128628Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.655168829Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.674305082Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.674360884Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.695699925Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.695755069Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.71487953Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.714927194Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.73328715Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.733331392Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.75274864Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.752794683Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.772087802Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.772134669Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.790761042Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.790806129Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.810097373Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.810141352Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.829374046Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.829479606Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.847872553Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.847929564Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.848088062Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.84824716Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.944977404Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945030852Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945035755Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945039449Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945046424Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945052818Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945058124Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945063487Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945069741Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945074466Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945079369Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945085603Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945090572Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945095879Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945100706Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945116598Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945121642Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945125101Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945129864Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:44.945224395Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"b45cd89e-ecdd-4615-9577-358bf327e09c","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized)]","latency_us":486989,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:44.945462085Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"830.505925ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b45cd89e-ecdd-4615-9577-358bf327e09c\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b45cd89e-ecdd-4615-9577-358bf327e09c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.779311638Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.779875447Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.779883313Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.803589005Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.803640926Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.823653136Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.823708497Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.842230996Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.842268319Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.861323513Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.861365335Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.879522854Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.879563205Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.900383288Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.90042649Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.919516958Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.919560291Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.939481788Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.939532481Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.958594296Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.958632228Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.979609761Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:45.979649194Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.000618828Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.000663604Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.020263698Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.020304461Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.039282084Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.039322238Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.059280333Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.059320938Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.077916458Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.077961018Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.09627106Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.096328295Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.115891648Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.115935355Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.133553053Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.133597922Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.152064027Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.152103394Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.170038868Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.17010712Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.17026513Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.170407438Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271334083Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271398191Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271404604Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271409882Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271416632Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.27142221Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271428792Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271434079Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271439189Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271444814Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271450111Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271455454Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271460808Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271465787Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271472237Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.27147774Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271496002Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271502509Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271506128Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:46.271588602Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"3cab751e-844b-493f-aa81-e4dfec96d41f","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized)]","latency_us":490584,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:46.271764764Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"877.411854ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3cab751e-844b-493f-aa81-e4dfec96d41f\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3cab751e-844b-493f-aa81-e4dfec96d41f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.152920171Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.153393284Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.153403662Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.176907566Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.176970952Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.197485156Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.197535717Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.218361512Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.21840508Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.23790548Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.237952675Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.256310643Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.25635256Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.276996321Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.277042805Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.297042101Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.297093843Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.317411588Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.317453842Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.338991974Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.339032344Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.359305244Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.35934941Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.379063421Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.379116419Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.398571907Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.398629996Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.418492305Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.418532197Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.437956584Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.437995913Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.458040598Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.45808274Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.478321714Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.478360761Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.496747144Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.496785179Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.51751015Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.517614598Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.540682097Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.540727877Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.56254161Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.562593398Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.562822964Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.562929476Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.6567215Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656769088Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656797167Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656804574Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656809759Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656816171Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656821384Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656824927Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656829962Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656833665Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656837088Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656840752Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656844166Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656849125Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656852847Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656857469Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656862719Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656872891Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.656876435Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:47.656984756Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"5bc25f58-edc1-4358-a591-01aa714b10ba","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized)]","latency_us":502384,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:47.657168876Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.534691598s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5bc25f58-edc1-4358-a591-01aa714b10ba\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5bc25f58-edc1-4358-a591-01aa714b10ba\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.197805327Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.198359466Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.198393948Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.22163146Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.221673209Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.245259245Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.245296578Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.264768825Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.264807039Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.282786854Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.282824111Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.302655078Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.302690967Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.322065641Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.322101426Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.340039089Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.340075399Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.359509884Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.359546381Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.378977903Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.37901292Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.398105804Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.398145357Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.417392126Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.417427415Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.437348287Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.437391075Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.456945123Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.456980477Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.476957827Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.476993415Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.497937141Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.497975973Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.517803979Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.517907185Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.538503715Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.53855408Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.558677868Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.558714431Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.57785427Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.577891724Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.599314012Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.599357935Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.599595795Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.599716483Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695843203Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.69589892Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695908192Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695926558Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695934049Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695940237Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695946453Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695950784Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695956924Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695961256Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695965466Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695970632Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695976697Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695982857Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695990225Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.695996198Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.696001992Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.696006295Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.696022964Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:49.69606895Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"cc8d4c0b-8454-428d-a2cc-0e8cde1c6481","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized)]","latency_us":496564,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:49.696241408Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"682.573966ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cc8d4c0b-8454-428d-a2cc-0e8cde1c6481\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cc8d4c0b-8454-428d-a2cc-0e8cde1c6481\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.381387623Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.381816795Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.381823714Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.404027443Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.404083339Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.423920754Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.423976995Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.444251684Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.444373218Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.463697839Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.463738303Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.483499531Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.483545113Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.502987078Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.503035379Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.521962865Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.522011336Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.541839112Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.541884516Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.562064392Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.562109263Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.58159031Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.58163234Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.600899743Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.600951316Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.620073541Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.620124459Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.639442519Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.639482589Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.658596287Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.658641288Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.677563061Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.677608728Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.696799063Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.696838346Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.715866666Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.71591402Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.735061922Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.735172766Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.754784038Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.754842642Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.774585326Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.774635436Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.774806085Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.774914079Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.871935759Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.871994007Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872001572Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872006644Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.87203057Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872036186Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872040012Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872043696Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872047268Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872050802Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872054393Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872057909Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872062521Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872067743Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872074363Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872078113Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872081375Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872084628Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.87208941Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:50.872115979Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"b7d750d4-e1ff-4142-8fc0-8baa83ad5ec3","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized)]","latency_us":489187,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:50.872295768Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"2.214735206s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b7d750d4-e1ff-4142-8fc0-8baa83ad5ec3\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b7d750d4-e1ff-4142-8fc0-8baa83ad5ec3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.094057801Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.094593284Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.094641352Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.116269265Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.116320267Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.137846044Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.137897009Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.15658963Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.156637913Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.175475528Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.175526745Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.194872606Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.194914485Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.213080868Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.213125016Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.232815509Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.232857323Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.252575357Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.252632059Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.273117563Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.273160361Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.294633994Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.294685483Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.314623735Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.314670433Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.333767554Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.333813324Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.354235593Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.354298539Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.374870129Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.374915608Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.393574236Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.393625388Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.412623922Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.412675243Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.431825011Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.431864997Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.453077338Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.453123399Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.471863462Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.471912852Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.492338896Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.492462513Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.492625662Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.492727299Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.58596566Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586011722Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586017122Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586022859Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586028775Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586043316Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586048773Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586055091Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586060285Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586065385Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586077986Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586083199Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586089274Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586094027Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586097514Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586100786Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586105371Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586109065Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586112543Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:53.58618372Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"f1b68c57-5cb9-4c21-8f22-bb95c1397dc0","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized)]","latency_us":490498,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:53.586357594Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"2.054333757s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f1b68c57-5cb9-4c21-8f22-bb95c1397dc0\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f1b68c57-5cb9-4c21-8f22-bb95c1397dc0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.647867144Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.648957795Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.64897377Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.671465392Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.671511246Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.69216325Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.692276707Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.713223087Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.713337546Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.733096949Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.73315667Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.75349754Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.753538285Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.771997216Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.772037061Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.790799076Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.790843646Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.813229725Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.813270245Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.832387756Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.832439855Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.851794602Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.851845059Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.87163707Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.871746111Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.892970888Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.893012842Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.912351483Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.912403224Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.930392206Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.930430945Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.948324408Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.948363972Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.968342901Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.968383467Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.989946626Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:55.98998861Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:56.011429339Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:56.011484156Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:56.030811059Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:56.030857814Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:56.050876965Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:56.050927227Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:56.051086109Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:56.051256297Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145733009Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145791361Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145797398Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145804092Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145809333Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145814733Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145819992Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145843796Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145847678Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145852918Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145856894Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145860438Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145863916Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145867394Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145872578Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145876403Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145883032Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145888028Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.145893158Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:56.145952774Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"1f184525-117d-4cfd-9ca9-c1075311e5b8","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized)]","latency_us":496458,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:56.146158664Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"2.876012879s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1f184525-117d-4cfd-9ca9-c1075311e5b8\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1f184525-117d-4cfd-9ca9-c1075311e5b8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.031028251Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.031532707Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.031545533Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.055830859Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.055876658Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.076680903Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.076731004Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.095449556Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.095500839Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.114836868Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.114890861Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.135491905Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.135532612Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.155166251Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.155211767Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.175106829Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.175154053Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.197080452Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.197129157Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.218747391Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.218804861Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.239138107Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.239188921Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.259475244Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.259530201Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.278773538Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.278822205Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.298504577Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.298559564Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.317734908Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.317786125Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.337724444Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.337773261Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.35852846Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.358579002Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.377426963Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.377481012Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.39788377Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.397930749Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.418631187Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.420993363Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.437369871Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.437426039Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.437603709Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.43771999Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533245644Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533295474Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533305234Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533309846Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533314262Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533319371Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533323534Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533333313Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533348913Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533366379Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533371517Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533375651Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533380873Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533449782Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.53346078Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.53346512Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533468505Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533471898Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533475058Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:20:59.5335403Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"4fe31a22-1798-45d9-9f40-fc3fa6305f02","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized)]","latency_us":500848,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:20:59.533721486Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"3.617705798s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4fe31a22-1798-45d9-9f40-fc3fa6305f02\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:20:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4fe31a22-1798-45d9-9f40-fc3fa6305f02\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.162814656Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.163347257Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.163368014Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.186927724Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.186971881Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.208320093Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.208360567Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.22860182Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.228642245Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.249628932Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.249669818Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.270262491Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.270370124Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.29114135Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.291187391Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.31048175Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.310519805Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.329047103Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.329083629Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.350010828Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.350058848Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.370738365Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.370775866Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.390812083Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.390852903Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.409850298Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.409889861Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.429084533Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.429122652Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.450245083Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.450294396Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.472345655Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.472386139Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.492118191Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.492153414Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.514417132Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.514455908Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.535318276Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.53535693Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.556391804Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.556438267Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.577041022Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.577093243Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.577354652Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.577478875Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677008844Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677085815Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677093971Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677106861Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677112919Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677125903Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677137247Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.67715012Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677155754Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677181639Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677200605Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677209587Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677215381Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677222871Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677228478Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677244453Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677251785Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677258882Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677267788Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:03.677359356Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"06fd1c38-d301-4f8a-97b7-a87ab3ae8208","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized)]","latency_us":512877,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:03.677647579Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"4.726115793s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"06fd1c38-d301-4f8a-97b7-a87ab3ae8208\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:21:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"06fd1c38-d301-4f8a-97b7-a87ab3ae8208\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.417236724Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.417729806Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.417757782Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.440710776Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.440805982Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.461645092Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.461703096Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.481077505Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.483473964Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.499811218Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.499860494Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.518780468Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.51884797Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.536527292Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.536580093Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.555108791Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.555172628Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.574063293Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.574114097Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.592157224Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.592211048Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.611816086Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.611877345Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.629945024Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.630001453Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.648169198Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.648222618Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.667027547Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.66708544Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.685364218Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.685419569Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.703373667Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.703431925Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.721624968Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.721688419Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.739537939Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.739601409Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.757453333Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.757507325Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.777555326Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.779768393Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.795301768Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.795374998Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.795566535Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.795710379Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887625706Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.88767006Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887683325Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887689833Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887694473Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887698776Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887702826Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887707176Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887711414Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887715164Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887725543Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887730165Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.88774194Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887747013Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887751381Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887760644Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887764994Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887769306Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.887773647Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:08.887900344Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"daed0e8f-89ee-40d8-8a49-9a77acdb7438","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized)]","latency_us":469149,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:08.888114756Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"5.580989s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"daed0e8f-89ee-40d8-8a49-9a77acdb7438\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:21:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"daed0e8f-89ee-40d8-8a49-9a77acdb7438\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:09.935784685Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52308"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:09.935884869Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"43e26eb9-fbe0-4fe8-849b-2c0656f368d0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52308","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.486009124Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.486619605Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.48662793Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.509934506Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.509975682Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.531241498Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.531323897Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.550334733Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.55036958Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.570503219Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.572795147Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.590399369Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.590448626Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.609868421Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.60991135Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.63246836Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.632531868Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.653955603Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.654017348Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.675095308Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.675150809Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.695376462Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.695421876Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.714109953Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.714163316Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.732267224Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.73231637Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.751071091Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.751111864Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.769539847Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.769578417Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.788299045Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.788347712Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.807347418Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.807389128Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.826063033Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.826104978Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.845106607Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.845142898Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.863323253Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.863362282Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.88152359Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.881579654Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.881810332Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.881960749Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978778617Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978856825Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978870898Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978877329Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978892863Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978903682Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978927027Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978937395Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978971559Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978988238Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.978997782Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.979005685Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.979015155Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.979025055Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.979032451Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.979037609Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.979044668Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.979050124Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.979055984Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:14.979149708Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"1bbc7b1b-2661-4c25-af9a-adf9dab1cabb","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized)]","latency_us":491450,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:14.979478212Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"5.135476024s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1bbc7b1b-2661-4c25-af9a-adf9dab1cabb\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:21:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1bbc7b1b-2661-4c25-af9a-adf9dab1cabb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.12960826Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.130434568Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.130450055Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.153117539Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.153182313Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.174033013Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.174086292Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.193427042Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.193476177Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.212619923Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.212656533Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.231420684Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.231453797Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.249997231Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.250053445Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.268795458Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.268832228Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.287909046Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.287945638Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.306939635Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.306977934Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.326340075Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.32637796Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.345328327Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.34537451Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.364944834Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.364980779Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.383836068Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.38387297Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.40287646Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.40292778Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.421506349Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.421545856Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.440709738Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.440754167Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.459412348Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.459443887Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.478444527Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.47847944Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.497112801Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.497160052Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.515137299Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.515178287Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.515395259Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.515523176Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609720695Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609794169Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609803497Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609812123Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609820607Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609827188Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.60983436Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609842789Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609849792Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609866978Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609877759Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609898703Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609907019Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609919282Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609927822Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609934067Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609938894Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609945345Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.609954186Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:20.6100305Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"3eab83d5-8047-42aa-b2c4-174a84143f03","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized)]","latency_us":478603,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:20.610313909Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"3.489561213s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3eab83d5-8047-42aa-b2c4-174a84143f03\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:21:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3eab83d5-8047-42aa-b2c4-174a84143f03\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.109778384Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.110405936Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.110419118Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.132893211Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.132937735Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.153342795Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.153392081Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.172907063Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.172942388Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.191185651Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.191225712Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.208739651Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.20885786Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.226287879Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.226326225Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.244708767Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.244764729Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.262705142Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.262740262Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.280745632Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.280784952Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.299782869Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.299839374Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.318595813Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.318635385Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.337244902Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.337281118Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.356121684Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.356160039Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.37517619Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.375218867Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.39514457Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.395180393Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.414595531Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.414661937Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.433522476Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.433559002Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.454087649Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.454131957Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.47316802Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.473220522Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.491905418Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.491945047Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.492135187Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.492258621Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586622185Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586693437Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586701884Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586707987Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586713912Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586723335Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586729344Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586734931Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586740463Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586747063Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586771176Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.58677847Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.58684894Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586866565Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586872368Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586876353Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586882035Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586886094Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.586890172Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:24.586969749Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"c89edc9f-9bd0-414a-9514-77b29cc8f5af","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized)]","latency_us":475471,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:24.587180627Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"4.087787662s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c89edc9f-9bd0-414a-9514-77b29cc8f5af\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:21:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c89edc9f-9bd0-414a-9514-77b29cc8f5af\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.686943958Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.687425414Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.687431499Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.709944259Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.709988978Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.731680692Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.731726903Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.751702606Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.751740154Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.769762921Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.769799353Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.785891873Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.785991797Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.80220273Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.802243794Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.821626547Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.821670797Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.839678049Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.839729912Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.858218451Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.858250814Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.876905041Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.876958967Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.894080004Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.894116015Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.912474191Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.912511045Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.930257789Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.930364929Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.948812624Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.948850547Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.967091462Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.967131757Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.985922878Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:28.985959246Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.003992733Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.004043106Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.024811813Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.024844552Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.043017447Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.04304712Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.061325785Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.06138259Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.061577642Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.061771896Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155380295Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155467109Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155476981Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155496313Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155506204Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155517801Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155529642Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155539317Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155550408Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155559568Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155589119Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155599272Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155622036Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155632676Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155652927Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155661815Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155669587Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155679309Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.155686988Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:29.155743107Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"77e261c1-7405-41b6-8673-185d1f09eadd","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized)]","latency_us":467262,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:29.156015936Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"5.930776568s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77e261c1-7405-41b6-8673-185d1f09eadd\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:21:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77e261c1-7405-41b6-8673-185d1f09eadd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.103130422Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.104358094Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.104372449Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.130901365Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.130953726Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.154594688Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.154643214Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.177232311Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.17728709Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.199662367Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.199711493Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.223348697Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.223406617Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.245807094Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.245857092Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.269151194Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.269280394Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.290347481Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.290395746Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.313433611Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.313487595Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.339104787Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.339159183Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.364251375Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.364310337Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.388290767Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.388340091Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.41348361Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.413538821Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.438968194Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.439019065Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.463509558Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.463575935Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.488307681Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.488353443Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.511007205Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.511057156Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.536529264Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.536586866Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.561743212Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.561794505Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.587306158Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.587375029Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.58757368Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.587732788Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699060763Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699131564Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699140761Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699151768Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699158528Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699164012Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699170725Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699176528Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699194894Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699205263Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699215473Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699225157Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699233164Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699255608Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699260108Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699286771Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.69929565Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699306038Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699312029Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:35.699439445Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"26dddc0d-4e03-47c9-8f8e-27b899d0a992","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized)]","latency_us":594362,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:35.699663674Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"4.778500562s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"26dddc0d-4e03-47c9-8f8e-27b899d0a992\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:21:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"26dddc0d-4e03-47c9-8f8e-27b899d0a992\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.047080661Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64745"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.047179569Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"c05180f3-6cde-478f-81a5-380d5d2cfcfc","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64745","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.491311473Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.491726994Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.491733022Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.513381482Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.51343133Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.531457482Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.531503796Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.551099364Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.551133583Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.569137367Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.569171783Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.587261252Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.587314748Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.605511673Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.60556783Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.623274532Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.62332544Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.640989903Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.641021779Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.658861367Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.658897471Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.677315396Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.677369483Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.696630073Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.696753657Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.714475188Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.714523442Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.732299445Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.732347614Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.75023236Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.750291481Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.768815411Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.768925025Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.786409693Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.78646647Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.804605741Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.804668696Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.821758436Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.821807815Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.839324928Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.839371561Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.856052203Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.856170455Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.856349785Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.856486589Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946587828Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946629942Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946636738Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.94664347Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946647867Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946653061Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946668886Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946675262Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946682003Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946687403Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946693628Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946709884Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946715022Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946725129Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946730519Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946734579Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946739576Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946743504Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.946747527Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:40.94682251Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"8a57c23f-2bed-43da-96dd-83f88b50ad93","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\", TimeNotSynchronized), (\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\", TimeNotSynchronized), (\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\", TimeNotSynchronized), (\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\", TimeNotSynchronized), (\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\", TimeNotSynchronized), (\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\", TimeNotSynchronized), (\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\", TimeNotSynchronized), (\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\", TimeNotSynchronized), (\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\", TimeNotSynchronized), (\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\", TimeNotSynchronized), (\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\", TimeNotSynchronized), (\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\", TimeNotSynchronized), (\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\", TimeNotSynchronized), (\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\", TimeNotSynchronized), (\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\", TimeNotSynchronized), (\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\", TimeNotSynchronized), (\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\", TimeNotSynchronized), (\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\", TimeNotSynchronized), (\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\", TimeNotSynchronized)]","latency_us":454044,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:21:40.947047468Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"11.931497086s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8a57c23f-2bed-43da-96dd-83f88b50ad93\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:21:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8a57c23f-2bed-43da-96dd-83f88b50ad93\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 711ccb54-227e-476e-9bc7-c738bae2623e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ea42831f-f0e1-4084-840f-aa951f986640, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82d8d423-6846-428f-8eb7-7d87943ccbca, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 7d50ad70-95f6-4e14-a4bc-bce0b176f6e1, kind: Service { id: 4933dd3d-baa4-43a7-b488-a6e5fdad89d5 }, name: Name(\"ntp-4933dd3d-baa4-43a7-b488-a6e5fdad89d5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 117])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35cba43-078c-4794-99ca-4255942899ac, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: 760e9943-67c9-4971-982a-df71813585c0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: a659e3c3-fb10-4e94-881b-2120466ce2b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 563f58c1-d407-4137-bc76-479e49aec380, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: cd916a02-7494-435c-a03d-4788b80a4b8f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: e440ec47-0611-4a7b-a40f-a3a9858d9733, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: af1c3446-0a0d-4d20-8b27-21e6fd6e6326, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 13517379-cc23-4e5c-a042-6fc9b8934357, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External } } } }, OmicronZoneConfig { id: fa50666f-58a0-483d-be64-fa945e831afc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External } } } }, OmicronZoneConfig { id: bd7c537f-b395-4fac-b80b-c5f64aec5175, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External } } } }, OmicronZoneConfig { id: 63b23166-4f95-4eb8-ad60-a228d77b8860, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External } } } }, OmicronZoneConfig { id: c2f00f12-490d-4063-bbdf-ba3339526aca, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External } } } }, OmicronZoneConfig { id: d51b3a20-a225-4ec7-923b-909364c7c31a, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External } } } }, OmicronZoneConfig { id: 5e26c358-ec8d-4c32-994c-352789460794, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External } } } }, OmicronZoneConfig { id: 314ccbfc-4da6-468c-890e-832bbc1bb521, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External } } } }, OmicronZoneConfig { id: cb89319f-4814-47d4-b860-8be4b528d881, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.911501727Z","hostname":"buskin","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.912084972Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.912118536Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 711ccb54-227e-476e-9bc7-c738bae2623e, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.931924641Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.931962349Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ea42831f-f0e1-4084-840f-aa951f986640, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.947907577Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.947946437Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 82d8d423-6846-428f-8eb7-7d87943ccbca, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.966806748Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.966842018Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e35cba43-078c-4794-99ca-4255942899ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.984209807Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:52.984255961Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 760e9943-67c9-4971-982a-df71813585c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.004409267Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.004444602Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70323a7d-e09e-4fb2-b8a6-3cb5eaff9000, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.024870995Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.024926478Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4712897e-ce28-4ed0-ae00-c454b60a50d5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.04296419Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.0429984Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cd4c53d6-b1da-42ff-b18a-4ad3b7218a9d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.060414461Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.060451943Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a659e3c3-fb10-4e94-881b-2120466ce2b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.080665171Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.080779538Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8eb46fe3-79ce-42d3-8484-9f04007321b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.098945556Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.099004048Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563f58c1-d407-4137-bc76-479e49aec380, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.117394325Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.117432134Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 13517379-cc23-4e5c-a042-6fc9b8934357, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d954909-2e42-44b1-96d9-f536366f6758, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.134992028Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.135022873Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fa50666f-58a0-483d-be64-fa945e831afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c6b0894-9c77-422a-8363-2705812a672c, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.153406801Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.153457981Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd7c537f-b395-4fac-b80b-c5f64aec5175, dataset_name: DatasetName { pool_name: ZpoolName { id: af9b6346-5d28-46df-baea-640dad9b76c3, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.171595791Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.171637229Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63b23166-4f95-4eb8-ad60-a228d77b8860, dataset_name: DatasetName { pool_name: ZpoolName { id: ae4e2e29-ff41-46eb-978b-335ffe5cd37e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.190394487Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.190456645Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c2f00f12-490d-4063-bbdf-ba3339526aca, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eb310e9-0d98-4a07-821c-54f35a32dc4a, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.20884585Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.208885489Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d51b3a20-a225-4ec7-923b-909364c7c31a, dataset_name: DatasetName { pool_name: ZpoolName { id: 494a3a1c-f913-4e4b-8eec-202ec6c5bfd4, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.227607249Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.227657931Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5e26c358-ec8d-4c32-994c-352789460794, dataset_name: DatasetName { pool_name: ZpoolName { id: 00b9d2d0-59a4-4682-aa18-072c6202c0c8, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.244690965Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.244728269Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 314ccbfc-4da6-468c-890e-832bbc1bb521, dataset_name: DatasetName { pool_name: ZpoolName { id: 70a16060-4587-476f-9772-16ea3a5a39aa, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.26337726Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.263420076Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cb89319f-4814-47d4-b860-8be4b528d881, dataset_name: DatasetName { pool_name: ZpoolName { id: 761d3524-c333-4332-9563-7d1ceb36709e, kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.28170393Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.281759019Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.282002973Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.282127777Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.385847108Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.410452741Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.410760666Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.41078222Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.424228472Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.424365379Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.43759583Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.437874429Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.450327082Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.450469839Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.463342625Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.640313282Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.640369242Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.640377006Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.640405938Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.640410457Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.640414179Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.640418922Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.640422635Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.640947724Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.645571892Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.650437773Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.65517419Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.660207208Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.665655269Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.826079232Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.831014419Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.83588917Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.840386235Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.844979532Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.849577541Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.855077468Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.860039415Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.864458298Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.869398636Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.875048102Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.880679304Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.885988115Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.891487113Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.898346703Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.904297279Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.910301526Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.915795785Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.921728234Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.926910304Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.933191926Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.938260204Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:53.981778564Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.023451904Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.05918613Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.094176335Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.125908307Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.127977589Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.127994746Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.183441617Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 160, 169]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.183515831Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1285","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.185240225Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.185250426Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.233001794Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 132, 230]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.233076036Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1285","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.234664968Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.234678234Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.287327685Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 153, 236]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.28738779Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1285","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.289569224Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.289578834Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.336986876Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 140, 131]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.337113426Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1285","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.338815661Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.33882273Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.391537305Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 166, 2]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.391596763Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1285","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.392769934Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.392822904Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.392927692Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.392946104Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.393009153Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.393019447Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.394010283Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.394022376Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.394324152Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.397161659Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.397260079Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.398651462Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.399879238Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.399885107Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.427380714Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.427470014Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.427562303Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.480053687Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.480348136Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.486918123Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Configuring new Omicron zone: oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.513354477Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.518747753Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.52511094Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Installing Omicron zone: oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.525238819Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.561385545Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.56761212Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.58648342Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.602681629Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.626930115Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:21:54.649467047Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:05.081784745Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:05.280655563Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:05.379876805Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:05.590271815Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:05.591452105Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.012046988Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.226523007Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.227439015Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.228398376Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.229509097Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.22967623Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.443504029Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.451281779Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.45155858Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.458005747Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.464506929Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.470715236Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.47821855Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.47832329Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.484597653Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.791814275Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:06.792620106Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:08.554257336Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:08.554321734Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:08.554340391Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:08.559914395Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 67.019838ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.622753439Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 33.718615ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.631032116Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 59.482152ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.639375416Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 44.290517ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.648297816Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 63.085948ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.656906575Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 74.356222ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.66453805Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 47.961483ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.671968285Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 65.676896ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.679042376Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 93.116466ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.686596093Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 66.363192ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.694855047Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 70.156815ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.703516754Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 71.988097ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.711983222Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 114.069506ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.720491072Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 139.050083ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.728685234Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 142.443111ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.737645218Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 140.183759ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.74577726Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 78.926929ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.757394249Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 126.783248ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.765563223Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 55.400782ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.772700244Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 55.190854ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.779940095Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 144.966992ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.789383409Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 93.749633ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.856428484Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 179.523061ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.867604373Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 144.241357ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.878710899Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 262.317266ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.888240147Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 101.736708ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.896135989Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 278.400889ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.905565925Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 60.392927ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.913037799Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 129.377531ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.922229148Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 100.775581ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.93148561Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 275.252927ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.942698465Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 276.514148ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.952653197Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 169.228588ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.963273836Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 263.998356ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.974476837Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 173.533646ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:08.986476682Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 167.535529ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.011068418Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 431.796374ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.033761073Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 476.940745ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.042774917Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 584.896294ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.057534913Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 259.742812ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.065623825Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 480.297905ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.14385466Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 466.781678ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.160916877Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 568.481061ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.171463148Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 597.56072ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.191239552Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 378.00546ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.200233194Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 240.575769ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.230338344Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 309.787268ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.239413699Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 231.746844ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.248682011Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 901.862405ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.337223602Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 450.50676ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.480728735Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 912.828054ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.493394535Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 594.83205ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.503773306Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.178214714s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.532699245Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 829.622101ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.563162689Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 617.977799ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.590649587Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 1.099878491s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.6366137Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 890.246779ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.648490873Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.027693055s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.660217267Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.166478344s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.753079293Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 735.062997ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.802673622Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 1.379355205s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:09.946902956Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 778.990235ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.11069187Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 1.488014333s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.22131972Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 1.153305076s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.252853478Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.461156628s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.404285073Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 1.129867343s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.415863685Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:10.475903356Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48311"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:10.476639387Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"c6b32bf3-f578-45af-a356-a1dea01ed563","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48311","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":467,"response_code":"200"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.180701454s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.546555242Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 794.364362ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.553237039Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.162384922s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.69701841Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.262151682s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.720054291Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 763.714891ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.744880882Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 1.417790738s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.898436657Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.127014951s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:10.929922373Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 1.342575579s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:11.336271934Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 1.174623439s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:11.356080625Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 748.950899ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:11.414530672Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 1.357564336s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:11.516591403Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 1.065933184s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:11.552942485Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 1.478262574s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:11.719921532Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.011324745s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:11.736286257Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.152698496s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.038310294Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 844.844212ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.254269795Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.181479916s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.607476226Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 1.023583112s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.634757796Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.444388044s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.663479055Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 626.969075ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.690227309Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 1.337931056s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.725285745Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 785.532927ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.746348841Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 783.175249ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.763710945Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 970.11031ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.782301172Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 979.193996ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:12.895702879Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.065107415s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.115066394Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 814.146316ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.171357161Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 582.510155ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.211935921Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 1.032576669s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.328648369Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 536.556521ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.544307442Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 1.448444697s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.559790503Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 1.125765987s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.6692631Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 730.575996ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.762800533Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 983.812205ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.799551976Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 1.003718389s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.808751992Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 803.941518ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.885618952Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.325659172s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:13.995655275Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 636.079884ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.072886844Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 1.10704557s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.089874777Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 623.688644ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.117231649Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.058809985s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.189201232Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 968.084865ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.371799274Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 597.037023ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.503564222Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 1.075538072s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.698852597Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 1.481859626s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.71798389Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.49151222s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.750201737Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 915.490179ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.806878094Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 1.428371118s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.814650451Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 728.482303ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:14.822022816Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 1.136693736s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:15.017952699Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.363924975s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:15.10995162Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 1.443595416s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:15.205275653Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.252623139s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:15.257107766Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 809.562203ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:15.330007725Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 1.471747772s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:15.349197279Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 576.136579ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:15.559770371Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.175522894s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:15.732615731Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 1.285347397s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:15.783090376Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 716.51786ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.146271915Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.069887357s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.155324567Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 1.482796802s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.164328617Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 760.193919ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.209044242Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.463107697s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.251600536Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 627.304839ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.259049894Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.485373841s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.482663538Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 531.110508ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.519420931Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 1.426339643s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.658203494Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 1.197610587s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.830938001Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 1.001843375s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.87184733Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 785.27525ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.894931283Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.36081872s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.917273675Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 866.251029ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:16.977752848Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.497207048s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:17.059885804Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 1.451625902s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:17.077178366Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.409008082s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:17.233819315Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 1.458129601s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:17.657080196Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 576.447187ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:17.689247392Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 748.43462ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:17.724416791Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 769.355436ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:17.85266233Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 1.384799724s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:17.882674336Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.095369492s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:17.976680502Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 1.212992397s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.038315968Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 1.086419569s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.093313939Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 1.363878729s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.274347858Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.104490993s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.28592536Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.096879608s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.48423639Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 568.056615ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.539377775Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.245773803s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.565890912Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 1.347432985s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.632513887Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.009379826s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.651504736Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:18.860880883Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:18.861430788Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:18.868241778Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:18.868924077Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 34.615635ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.933613798Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 63.190641ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:18.981065708Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:19.014294033Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:19.014774346Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 202.79155ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.06197547Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.18256325s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.092324815Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 842.007039ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.121096486Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 861.472176ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.132363279Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 32.694258ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.157278993Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 74.53486ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.167320052Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 1.303859122s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.189091998Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 101.833236ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.198389887Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 89.23912ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.252292938Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 714.069751ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.262481781Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 49.635704ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.270508245Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 1.075150276s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.278116196Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 534.179613ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.28505448Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 186.270268ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.308690818Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 55.395625ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.328094138Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 249.387781ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.350340497Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 216.160392ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.392226583Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.018969291s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.401568243Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:19.498864251Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 499.493639ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.507742618Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:19.507845511Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.209054429s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.594864463Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 542.151153ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.6245823Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 518.241167ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.655802929Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 965.877623ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.67989941Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.045738388s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.691966965Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 987.330072ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.827972491Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 1.053667243s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.841926019Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 64.809713ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.853501602Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:19.899105721Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:19.899170943Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 73.94577ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.927766799Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 725.699185ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.976121255Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 1.072313842s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.986639577Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 1.043573208s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:19.995210306Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 522.811046ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.003352413Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 212.831512ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.010697827Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 447.394475ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.019154242Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 60.410887ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.080532771Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 118.600264ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.150684314Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 1.131534091s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.176831081Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 1.075565782s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.186267734Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 520.592428ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.232172652Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 213.096436ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.281643719Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.059274268s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.290768133Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 1.334181933s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.367525919Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 818.330595ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.433857965Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 1.093729783s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.477223752Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 623.861764ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.503774351Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 264.136861ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.512933846Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 1.167685319s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.536623066Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 547.304158ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.657317175Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 871.34397ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.713516669Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 547.718347ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.752454223Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 1.17870537s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.769267491Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 798.181965ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.790707968Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 530.879076ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.817095002Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.273164552s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.831881894Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 842.33271ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:20.910870833Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 709.793419ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.051702953Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 795.093517ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.073273673Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 1.471139367s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.147065667Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 804.752043ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.224498122Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.268091805s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.264347071Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 1.026726057s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.276967003Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.269947126s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.31421582Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 1.473357959s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.326588174Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 818.17529ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.361557057Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 587.766459ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.375830684Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 712.285567ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.584539754Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 1.020436222s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.598251426Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 563.123992ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.610876717Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 1.131699464s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.715739745Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 1.159982376s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.728369267Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 1.242901518s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.766196809Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 1.470454193s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.778340547Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 1.490012847s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.882015213Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 934.86602ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.963402678Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 885.58961ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:21.977352237Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 1.466190271s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.042455849Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 548.255449ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.119792362Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 839.186122ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.188266557Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 883.43612ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.201916588Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 1.269508286s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.312368452Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 606.326134ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.32668014Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.317768267s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.546694734Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.079978432s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.599487854Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 708.443194ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.633320495Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 1.172728748s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.645851194Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.043871441s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.68021945Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 633.241515ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.820490805Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 1.491046399s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.864432678Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.11798s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.880064153Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 875.366712ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.904886904Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 504.689551ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.916992371Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 1.116226694s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:22.948704914Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 757.619445ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.022652152Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 965.703097ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.043545314Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.411247556s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.101516517Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 877.454993ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.263067224Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 979.025993ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.358789918Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 896.539167ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.388211307Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 1.243601129s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.435352409Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:23.451499629Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 1.178567315s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.469606613Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:23.469697685Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 869.495205ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.523461022Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 681.314701ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.596466904Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 845.392862ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.695628393Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.46242182s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.739899325Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 32.229291ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.776481024Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 739.442569ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.792262013Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 1.327773812s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.802699954Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 74.881629ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.821794885Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 1.363186644s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.835296751Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.321754467s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.878781381Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 217.229975ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:23.909602349Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.429455835s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.018438249Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 760.482489ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.037444348Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 1.421151714s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.085860355Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 551.976074ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.148302623Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 1.057408296s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.168095104Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 636.45835ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.290201073Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 1.270480978s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.302818474Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 1.091517551s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.349404801Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 1.229169142s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.369217572Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 848.524835ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.406977123Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.019355335s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.526904222Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 1.442861107s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.544427219Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.414993551s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.557319448Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 1.366454316s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.658678009Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 570.558355ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.690575367Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 1.182628804s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.70971715Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 1.423371882s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.810038292Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 618.561986ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:24.939692741Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 1.172661965s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.144599274Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 718.170775ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.214429688Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.299552209s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.229895208Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 526.74354ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.243022104Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 933.248845ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.256099958Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 1.45914288s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.270382325Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 1.454501628s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.282883653Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 1.154092939s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.460844817Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.112627718s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.478122548Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 794.05123ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.524952971Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 864.624497ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.559622136Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 1.471895329s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.573950315Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 1.154143983s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.585500196Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 789.485556ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.610336358Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.487213315s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.785328998Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 774.206631ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.907066426Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 1.363233822s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.946014061Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 660.704389ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:25.989543962Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 1.346755255s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.00274797Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 813.803974ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.038137015Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 826.625642ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.205090502Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 549.833924ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.244943631Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 827.706667ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.330650907Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 549.659976ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.342428595Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 633.085981ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.415855218Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.298254285s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.439773034Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 1.362847819s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.544334744Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 1.189733636s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.603547203Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 906.571871ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.627023536Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 998.129817ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.667173363Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 1.359846792s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.698209997Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326\") failed: Property not found. retry in 1.199671715s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.743506888Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 539.271784ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.758437683Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 766.103883ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.775414506Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 681.187271ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.809232837Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 548.432532ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.864249269Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 1.19649774s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:26.905513117Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 615.389191ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.045212326Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 858.954531ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.06010334Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794\") failed: Property not found. retry in 872.467776ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.069569084Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 908.794147ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.174947807Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 614.550073ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.284768854Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 635.949337ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.311356304Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a\") failed: Property not found. retry in 815.637292ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.321212468Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc\") failed: Property not found. retry in 949.81425ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.361060357Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 812.682393ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.424081331Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 507.846853ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.502125624Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175\") failed: Property not found. retry in 864.018456ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.545199333Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4\") failed: Property not found. retry in 1.40157717s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.560054986Z","hostname":"buskin","pid":1618,"zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881\") failed: Property not found. retry in 526.012031ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.675567301Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8\") failed: Property not found. retry in 1.312959727s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.687465282Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860\") failed: Property not found. retry in 1.399239159s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.750707052Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\") failed: Property not found. retry in 985.449898ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.813437128Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca\") failed: Property not found. retry in 1.162750581s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.919694586Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4\") failed: Property not found. retry in 781.910839ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.937649576Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f\") failed: Property not found. retry in 1.187165712s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:27.952263012Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521\") failed: Property not found. retry in 1.058381215s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:28.019076082Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0\") failed: Property not found. retry in 1.184002006s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:28.036215469Z","hostname":"buskin","pid":1618,"zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 1.218400801s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:28.076165771Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357\") failed: Property not found. retry in 1.129014505s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:28.103572374Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6\") failed: Property not found. retry in 713.195568ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:28.131856121Z","hostname":"buskin","pid":1618,"zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 633.773408ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:28.262543589Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733\") failed: Property not found. retry in 911.765234ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:28.929786161Z","hostname":"buskin","pid":1618,"zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 1.284307797s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:29.487547566Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 1.191186377s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:30.790554864Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 504.354272ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:31.994465423Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 1.275784762s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:32.509400826Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 808.3738ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:33.796272954Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 1.387253362s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:34.61494783Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 563.630345ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:36.014528293Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 936.144664ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:36.588884368Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 633.30536ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:37.535904626Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 936.362498ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:38.180207878Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 590.866697ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:39.127300917Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 519.144886ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:39.728647043Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 686.131689ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:40.258545847Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:40.508684233Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33282"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:40.50895537Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"0ca62d66-c6d3-4331-bc00-43b0aabd5485","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33282","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":85,"response_code":"200"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 796.080924ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:40.955965051Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 625.808231ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:41.762178319Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380\") failed: Property not found. retry in 979.665547ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:22:42.397881192Z","hostname":"buskin","pid":1618,"zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407601805Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_external_dns_8eb46fe3-79ce-42d3-8484-9f04007321b8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407657186Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_bd7c537f-b395-4fac-b80b-c5f64aec5175"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407663326Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_fa50666f-58a0-483d-be64-fa945e831afc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407668186Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_nexus_51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407672627Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407677047Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_nexus_8e55875e-d1c7-4353-bb0e-8bd66a3580e4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407683787Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_pantry_af1c3446-0a0d-4d20-8b27-21e6fd6e6326"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407688477Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_external_dns_a659e3c3-fb10-4e94-881b-2120466ce2b6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407695467Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_13517379-cc23-4e5c-a042-6fc9b8934357"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407714128Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_nexus_0a8b454f-8da6-4dd8-942c-effe3a8487e0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407718998Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_63b23166-4f95-4eb8-ad60-a228d77b8860"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407723518Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_d51b3a20-a225-4ec7-923b-909364c7c31a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407731638Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_314ccbfc-4da6-468c-890e-832bbc1bb521"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407736138Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_pantry_cd916a02-7494-435c-a03d-4788b80a4b8f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407740668Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_pantry_e440ec47-0611-4a7b-a40f-a3a9858d9733"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407744789Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_cb89319f-4814-47d4-b860-8be4b528d881"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407749209Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_clickhouse_563f58c1-d407-4137-bc76-479e49aec380"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407753419Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_crucible_c2f00f12-490d-4063-bbdf-ba3339526aca"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.407760349Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005","zone":"oxz_oximeter_2af5a5df-6e1e-4695-b84f-2a6edd1c1ac4"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.409100416Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.410705781Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.414121626Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"c1d5fa6b-6750-4827-b21e-21e70ece4669","remote_addr":"[fd00:1122:3344:101::1]:53174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50456987,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.438296729Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1230"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.438549166Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.43868676Z","hostname":"buskin","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.442806834Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:677"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.444224624Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:685"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.484722891Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:816"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"772ddc8e-1c76-4d05-8e34-69d50676452f\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"772ddc8e-1c76-4d05-8e34-69d50676452f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.637827532Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"655bf0bb-3e6b-4cc5-b118-eea4cb4e352c\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"655bf0bb-3e6b-4cc5-b118-eea4cb4e352c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.752455862Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e995cf1c-0c09-4b9a-87ef-76189881d115\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e995cf1c-0c09-4b9a-87ef-76189881d115\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:43.97372907Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"37f5ec01-1a58-44ad-b26b-25317124110f\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"37f5ec01-1a58-44ad-b26b-25317124110f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:44.206653343Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c5594691-b1df-4145-ba29-f9cd3274c2b7\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c5594691-b1df-4145-ba29-f9cd3274c2b7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:44.51661452Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6704d157-b564-4fa1-a59c-d865328a6f6b\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6704d157-b564-4fa1-a59c-d865328a6f6b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:44.881189097Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b0652bc7-ea93-45c7-bedc-665c3c5994d3\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b0652bc7-ea93-45c7-bedc-665c3c5994d3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:45.26180461Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e45e7e61-fae7-4c18-9899-37c4807ff2d1\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e45e7e61-fae7-4c18-9899-37c4807ff2d1\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:45.624478324Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ee3f7043-6c0e-4d8f-829c-77117ff2645c\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ee3f7043-6c0e-4d8f-829c-77117ff2645c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:45.959139308Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"327a1541-ec25-42da-b567-dea630f66f42\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"327a1541-ec25-42da-b567-dea630f66f42\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:46.334348881Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"afbfb227-9790-411f-969e-16944633296d\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"afbfb227-9790-411f-969e-16944633296d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:47.097418248Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8ecac43d-0ef3-4121-a0c2-9a64a3130561\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8ecac43d-0ef3-4121-a0c2-9a64a3130561\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:47.850555959Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d99777c1-145c-4457-839e-53e222bfbff9\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d99777c1-145c-4457-839e-53e222bfbff9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:48.347007937Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fd7f60e8-c972-477d-ad24-d967f3f28795\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fd7f60e8-c972-477d-ad24-d967f3f28795\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:49.371455978Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f2e159bb-77da-4cfe-84da-5034c657fe4a\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f2e159bb-77da-4cfe-84da-5034c657fe4a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:50.352015749Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"04ecaa8a-edc0-4e8f-85ed-e7a1a5b64006\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"04ecaa8a-edc0-4e8f-85ed-e7a1a5b64006\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:51.915797031Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"73a3e20f-4843-4e34-9426-589424f20988\", \"content-length\": \"124\", \"date\": \"Wed, 03 Apr 2024 22:22:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"73a3e20f-4843-4e34-9426-589424f20988\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:53.589551415Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:55.348194841Z","hostname":"buskin","pid":1618,"component":"NexusNotifierTask","sled_id":"1ee9e643-97de-413d-8b33-615f41c711a3","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:55.678652528Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:44378"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:55.680908921Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"f8258a8b-839f-4cba-a230-d97f0036ade7","remote_addr":"[fd00:1122:3344:101::b]:44378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1577,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.436868551Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:44437"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.439055312Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f2bf8558-3c4f-40ab-8317-d015a4fd68f9, kind: Service { id: 8eb46fe3-79ce-42d3-8484-9f04007321b8 }, name: Name(\"external-dns-8eb46fe3-79ce-42d3-8484-9f04007321b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f60f6dde-6446-46fe-a1a3-11cee72d82a3, kind: Service { id: a659e3c3-fb10-4e94-881b-2120466ce2b6 }, name: Name(\"external-dns-a659e3c3-fb10-4e94-881b-2120466ce2b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 131])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 211481ad-d25e-42ee-9a8a-1eaab1f31cec, kind: Service { id: 51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2 }, name: Name(\"nexus-51c8ee79-ce7f-49ba-8ad8-5c1c00a001d2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 7c50abb7-e764-48dd-9815-f90f1b10b5f8, kind: Service { id: 8e55875e-d1c7-4353-bb0e-8bd66a3580e4 }, name: Name(\"nexus-8e55875e-d1c7-4353-bb0e-8bd66a3580e4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 236])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 92ea2ee4-1d24-4d12-9e19-b96e42ca9de4, kind: Service { id: 0a8b454f-8da6-4dd8-942c-effe3a8487e0 }, name: Name(\"nexus-0a8b454f-8da6-4dd8-942c-effe3a8487e0\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.439114383Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.439149904Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.439165265Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.439171855Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.439189525Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.439202786Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.439221906Z","hostname":"buskin","pid":1618,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"81e0df6a-544a-499e-97c0-8f9fe4628a38","remote_addr":"[fd00:1122:3344:101::b]:44437","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2194,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.44044441Z","hostname":"buskin","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:883"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.736906671Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:47220"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.738371902Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.8252316Z","hostname":"buskin","pid":1618,"uri":"/inventory","method":"GET","req_id":"b8f4a2bb-f69e-4012-a06c-7e2dfd97c06d","remote_addr":"[fd00:1122:3344:101::b]:47220","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":88206,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.826168295Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.826233487Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.827739099Z","hostname":"buskin","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:22:56.828035807Z","hostname":"buskin","pid":1618,"uri":"/omicron-zones","method":"GET","req_id":"eb993dc0-7939-4502-aa1e-daad260b2d65","remote_addr":"[fd00:1122:3344:101::b]:47220","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2551,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.546961848Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.54703712Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"7b85a6e1-9064-4b19-abdf-3ce70f8a18cd","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33310","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.689858105Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59506"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.689931737Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:56622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.782389871Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"99600225-e8e1-438b-b650-d080b95eb983","remote_addr":"[fd00:1122:3344:101::a]:59506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92448,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.785331942Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"86bf67b4-fde7-4318-a224-e5f4ee0902c9","remote_addr":"[fd00:1122:3344:101::a]:59506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":436,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.808696673Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.808850337Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.8100275Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.813584529Z","hostname":"buskin","pid":1618,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.813766884Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"PUT","req_id":"712aa250-4d81-4c33-82dc-0965d3c68df4","remote_addr":"[fd00:1122:3344:101::a]:59506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6732,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.836551808Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"ffb5b67a-bf9b-48d8-9ba5-2d0f84aa1faf","remote_addr":"[fd00:1122:3344:101::c]:56622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":146478,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:10.836881957Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"f036b031-aaac-419e-8ec8-913770800b46","remote_addr":"[fd00:1122:3344:101::c]:56622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:11.058743312Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:35225"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:11.103620301Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"423b2eac-2ea7-4907-995f-893ae8eb2385","remote_addr":"[fd00:1122:3344:101::b]:35225","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44801,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:11.103852218Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"e0b5d09d-7df2-430c-ac59-204255e1673c","remote_addr":"[fd00:1122:3344:101::b]:35225","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:23.381626479Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:57652"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:23.388115629Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"472c3a5d-1ee1-4bf6-b7cd-bbe18be8370f","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6458,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:40.580865994Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61366"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:40.580958167Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"ee505a09-1212-4595-97fc-1136fc40e529","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61366","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:40.866679869Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:54306"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:40.916696881Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"f24da286-98da-4e67-9cf7-0ae9f51ee4c4","remote_addr":"[fd00:1122:3344:101::a]:54306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49976,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:40.916942678Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"ff2cb584-6c6b-4349-8dd9-afe4ae001461","remote_addr":"[fd00:1122:3344:101::a]:54306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:40.929507017Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:47499"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:40.973164623Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"5ee641e9-913c-4c88-9e58-57f125803919","remote_addr":"[fd00:1122:3344:101::c]:47499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43548,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:40.97343019Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"4fc8ba47-d17d-45f5-8485-b9feb9875ac6","remote_addr":"[fd00:1122:3344:101::c]:47499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:41.414102336Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:63974"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:41.458268415Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"683b9725-339f-475a-babf-13c49d1d1ec4","remote_addr":"[fd00:1122:3344:101::b]:63974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44119,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:41.459138589Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"bb4c3d4d-826c-47f7-a491-ee20c9882d4e","remote_addr":"[fd00:1122:3344:101::b]:63974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":691,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:23:53.383911312Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"6105753b-cc41-4599-8566-c5c5d9969176","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2419,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:10.6165705Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:10.616672853Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"cde693bc-76e8-4bc6-8e1b-78e1ab66a386","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59483","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:10.676609751Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37936"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:10.720971095Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"72359743-65e3-480e-9a81-74232513768d","remote_addr":"[fd00:1122:3344:101::c]:37936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44305,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:10.721109509Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34516"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:10.724296858Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"1148e1ac-cf8e-4269-a5d8-109678a4ef8f","remote_addr":"[fd00:1122:3344:101::c]:37936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3008,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:10.762429049Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"3b49e5fa-0233-41e6-a511-3d77bc378755","remote_addr":"[fd00:1122:3344:101::a]:34516","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41213,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:10.763283243Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"860a8677-994f-4ec0-a4ca-67743a0f8b2e","remote_addr":"[fd00:1122:3344:101::a]:34516","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":663,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:11.0933577Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:64968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:11.138718683Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"6ac8d104-c6f9-4003-80b0-bdfc34357883","remote_addr":"[fd00:1122:3344:101::b]:64968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45307,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:11.13896291Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"2dc909ea-fa16-49b5-81d6-648374d5844d","remote_addr":"[fd00:1122:3344:101::b]:64968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":79,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:23.381668631Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"2d738f98-dff8-41bf-8760-1bfb1044d086","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":731,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:40.649667973Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62626"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:40.649771006Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"abdcc9cd-305e-4ca7-8ecf-3fe33efae8c0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62626","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:40.767281496Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58697"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:40.815057736Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"0bb751b3-0a21-4550-81f4-bae04f591f7d","remote_addr":"[fd00:1122:3344:101::a]:58697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47724,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:40.816000552Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"e543cd82-c243-47d4-a21d-33904a0629c7","remote_addr":"[fd00:1122:3344:101::a]:58697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":791,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:40.975641016Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:54776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:41.023776736Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"47ac46f6-e1b1-46df-9215-2770dd18dab7","remote_addr":"[fd00:1122:3344:101::c]:54776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48063,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:41.024056873Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"d6a54082-cf8e-410f-a294-ae30a9f136fe","remote_addr":"[fd00:1122:3344:101::c]:54776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:41.36884254Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:64275"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:41.414378577Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"6789f518-eb47-44b0-a164-9193a691f5c0","remote_addr":"[fd00:1122:3344:101::b]:64275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45444,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:41.414659855Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"a7f8e193-d673-4639-9fdb-a4f7c1d3f4e8","remote_addr":"[fd00:1122:3344:101::b]:64275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:53.381889629Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"cf63e6e8-de74-4c7a-8598-ca7a55ed8b8d","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":806,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:24:57.408106631Z","hostname":"buskin","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:10.684112091Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63711"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:10.684192163Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"6fce4c05-5e27-4fb0-b4a7-bff74f813f9b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63711","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:10.684596835Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42864"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:10.731463159Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"b3927116-56cb-43ff-959d-46d7bcd73b39","remote_addr":"[fd00:1122:3344:101::a]:42864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46793,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:10.731753597Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"381bd409-1260-4865-aa32-b88b1a166426","remote_addr":"[fd00:1122:3344:101::a]:42864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:10.793690501Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:38125"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:10.841772489Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"6b64f940-10bf-4bac-ae96-03ae3bf2b9bc","remote_addr":"[fd00:1122:3344:101::c]:38125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47995,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:10.842012476Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"f236dce6-0649-4591-b79e-82726c852ed9","remote_addr":"[fd00:1122:3344:101::c]:38125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:11.083170778Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:62275"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:11.133076327Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"b755e128-3a70-4b77-a758-525c48c04f53","remote_addr":"[fd00:1122:3344:101::b]:62275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49808,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:11.133342345Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"4ada6dce-cfc2-4648-bc0e-7b4cd41fda4f","remote_addr":"[fd00:1122:3344:101::b]:62275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:23.381836097Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"f71b5b3b-1a29-4fe8-8f8b-6db7ddd60d65","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":486,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:40.716316367Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41724"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:40.717245243Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"64deca9a-8953-4af5-90a0-e91b7b610659","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41724","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":920,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:40.830368402Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63867"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:40.877760181Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"c914c67c-32b4-47fb-80ec-e75a730f74e3","remote_addr":"[fd00:1122:3344:101::c]:63867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47334,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:40.877889554Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:48807"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:40.881366111Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"ca5368b3-82ea-418b-bd07-b58eb60563d9","remote_addr":"[fd00:1122:3344:101::c]:63867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3230,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:40.925352736Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"fe3b48ad-47fe-4930-97ae-21fe32232171","remote_addr":"[fd00:1122:3344:101::a]:48807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47385,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:40.925654704Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"8a144dd4-48d7-4f7e-90d2-e6e8aeb7b419","remote_addr":"[fd00:1122:3344:101::a]:48807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:41.402800754Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:58553"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:41.450381599Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"95d844c1-0bdb-4523-b385-5c8a1053a2f4","remote_addr":"[fd00:1122:3344:101::b]:58553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47503,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:41.450636886Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"9ea01097-4848-4b6a-b597-0054eeb1a451","remote_addr":"[fd00:1122:3344:101::b]:58553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":84,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:25:53.381593751Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"2b3f6873-a352-48b0-830a-8077e60aa278","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":641,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:10.580769708Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34904"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:10.629459453Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"2935ae0b-30a5-406d-b9d0-3b353de31e90","remote_addr":"[fd00:1122:3344:101::c]:34904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48658,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:10.62970553Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"3d12307f-bcb7-4509-abcd-a8a8c76e4012","remote_addr":"[fd00:1122:3344:101::c]:34904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:10.662821491Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42383"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:10.706924259Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"c8483c38-de57-45c6-a6c9-5e27b51725c8","remote_addr":"[fd00:1122:3344:101::a]:42383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44023,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:10.707168276Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"88893640-9364-4c83-b0c6-85b58c6ebed0","remote_addr":"[fd00:1122:3344:101::a]:42383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:10.748554927Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39799"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:10.749273857Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"0866dc38-5d6a-4090-a2d3-9aeb14a5c35f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39799","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":669,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:11.25307322Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:45987"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:11.300199222Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"8790ab62-12de-45fa-baf3-707075275314","remote_addr":"[fd00:1122:3344:101::b]:45987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47052,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:11.300448909Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"f441cd5c-7249-4c03-adbf-be5aa8504d86","remote_addr":"[fd00:1122:3344:101::b]:45987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:23.381842001Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"1fe46548-2df2-47a8-af26-2e215677ebaf","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":878,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:40.781441167Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:40.782083755Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"de088b84-f0e6-4e4f-b451-87fe8351fbc5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56231","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":633,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:40.844927824Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:35851"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:40.896258162Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"4c9e0c7b-285b-4e70-aa1a-e7a59d0fc343","remote_addr":"[fd00:1122:3344:101::a]:35851","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51287,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:40.896387476Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52820"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:40.899847822Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"93619bd4-21e6-4b75-b282-f565601260af","remote_addr":"[fd00:1122:3344:101::a]:35851","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3210,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:40.942647444Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"c8c96124-43f4-4e6e-858d-df1099b642e4","remote_addr":"[fd00:1122:3344:101::c]:52820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46155,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:40.942973583Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"c5cb3b67-eeb7-4ac7-8f6b-40efdc504719","remote_addr":"[fd00:1122:3344:101::c]:52820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:41.449022848Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52447"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:41.493044853Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"28a9365c-5b78-4c48-9bca-b3a2834f5a31","remote_addr":"[fd00:1122:3344:101::b]:52447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43962,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:41.493318141Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"f3b38dc0-c932-4eb3-9b6f-add82859158f","remote_addr":"[fd00:1122:3344:101::b]:52447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":84,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:26:53.38162955Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"e18f880c-016b-4387-89a0-2c1cb63a1912","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":484,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:10.635777729Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:51368"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:10.682686845Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"2e8f29f2-d224-4a3d-bd25-7fbbccd20a90","remote_addr":"[fd00:1122:3344:101::a]:51368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46849,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:10.682820679Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:50387"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:10.685624707Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"49398a5f-e9dd-4864-b521-5c09f1e1ed3a","remote_addr":"[fd00:1122:3344:101::a]:51368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2634,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:10.727603045Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"58ccec9f-106b-41c6-8d91-a375f6ecc1a0","remote_addr":"[fd00:1122:3344:101::c]:50387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44699,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:10.727955105Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"a22a9ba4-468d-419a-908c-cf752be6f22a","remote_addr":"[fd00:1122:3344:101::c]:50387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:10.822844496Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:10.822932938Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"53fc26c0-eddd-4437-99c1-5cb37513c3f2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64344","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:11.198931674Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:44736"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:11.244908134Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"95a7d428-eed4-47f4-953d-0ad099d7a7d1","remote_addr":"[fd00:1122:3344:101::b]:44736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45912,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:11.24514986Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"ecc48bf5-aa8b-4b8c-bd86-d4974d17d519","remote_addr":"[fd00:1122:3344:101::b]:44736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":85,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:23.381701121Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"1b833611-1067-4e50-8de9-4bf284d3c5f1","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1066,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:40.824449806Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:57673"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:40.875928939Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"9c2d14e0-7b2b-42bd-8dbf-385127aeddfb","remote_addr":"[fd00:1122:3344:101::c]:57673","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51444,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:40.876104124Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:43621"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:40.876204047Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57068"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:40.876374742Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"de9a13eb-3388-4201-8668-a8984ae7712e","remote_addr":"[fd00:1122:3344:101::c]:57673","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":189,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:40.876392772Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"c7d8ac7c-82ac-4f48-a393-554fa4f650e9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57068","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":128,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:40.923125723Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"e9b8b0c9-bbcf-4788-a372-137156324184","remote_addr":"[fd00:1122:3344:101::a]:43621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46855,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:40.923514064Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"ffda5652-a71e-4cff-a0ad-6f841f8895b3","remote_addr":"[fd00:1122:3344:101::a]:43621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":156,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:41.467947777Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:35020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:41.513609758Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"e0bd9eee-a1d9-4fd3-930c-e2dc5719fc6e","remote_addr":"[fd00:1122:3344:101::b]:35020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45595,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:41.513846505Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"226b6cd5-dcdc-4698-8f65-4999677faad0","remote_addr":"[fd00:1122:3344:101::b]:35020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:27:53.382500896Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"6fb3e172-096c-4030-89ec-6dacf11a72dd","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1819,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:10.571748414Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:48192"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:10.619781591Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"a8636e72-9a18-42b9-a3e6-ca7d2f5035ff","remote_addr":"[fd00:1122:3344:101::c]:48192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47995,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:10.619907244Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:36751"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:10.623387731Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"2a820bdf-d668-4fba-a24e-638c24bbdd92","remote_addr":"[fd00:1122:3344:101::c]:48192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3258,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:10.664735452Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"d941d68c-138d-4f44-b66e-b7e4c1082a08","remote_addr":"[fd00:1122:3344:101::a]:36751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44720,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:10.665145213Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"63c3c685-fd42-4e6b-bdea-f1ce63a37440","remote_addr":"[fd00:1122:3344:101::a]:36751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:10.910494562Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63878"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:10.910596425Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"86a3580d-d0c4-4af3-869d-41fb4667c338","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63878","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:11.114436178Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:34754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:11.163763281Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"08008fa7-a55d-40a3-a924-bc40aa159e1a","remote_addr":"[fd00:1122:3344:101::b]:34754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49274,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:11.164033959Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"be1d9367-4510-4c69-8ebc-12afebcafc04","remote_addr":"[fd00:1122:3344:101::b]:34754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:23.381009374Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"f6a80abf-9e1c-4934-8773-21c040709f16","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":827,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:40.761640789Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60516"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:40.80766323Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"a4f89186-b006-431c-a7eb-d1ea8b748b9c","remote_addr":"[fd00:1122:3344:101::c]:60516","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45982,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:40.807835385Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39203"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:40.811848246Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"1fa59a15-ac14-4894-a277-e1dbb8aa2657","remote_addr":"[fd00:1122:3344:101::c]:60516","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3891,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:40.85365437Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"da3bdde0-d811-44ef-9a71-f263d80af227","remote_addr":"[fd00:1122:3344:101::a]:39203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45733,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:40.85401077Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"c6019658-18b0-4e71-8760-91e01c7e2694","remote_addr":"[fd00:1122:3344:101::a]:39203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:40.943643674Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51575"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:40.943733867Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"3a4932be-4414-48ee-a46e-04f41f59475d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51575","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:41.502453978Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:49613"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:41.550036672Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"b2b86b2c-3660-4074-a1bb-4c5212b0a604","remote_addr":"[fd00:1122:3344:101::b]:49613","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47540,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:41.55029339Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"a15e7d30-3179-48e3-bdf3-1d1b6c610200","remote_addr":"[fd00:1122:3344:101::b]:49613","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:28:53.380637933Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"64dee62b-4e9a-4c17-92b9-f51307f97d2a","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":644,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:10.539601839Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58887"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:10.585134987Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"bfc0ea4b-2e0b-48f7-bcbc-e2502cfa4d61","remote_addr":"[fd00:1122:3344:101::a]:58887","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45488,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:10.585482606Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"09baa1cc-f166-4252-a62a-cfb98e42950a","remote_addr":"[fd00:1122:3344:101::a]:58887","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:10.662758286Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52341"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:10.702688078Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"3da05712-3677-41ba-af60-2ff41996b9cb","remote_addr":"[fd00:1122:3344:101::c]:52341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":39834,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:10.703581503Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"31e5fa16-fd3f-44c8-9c5e-c71d22619fca","remote_addr":"[fd00:1122:3344:101::c]:52341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":636,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:10.975984915Z","hostname":"buskin","pid":1618,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37729"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:10.976115398Z","hostname":"buskin","pid":1618,"uri":"/baseboard","method":"GET","req_id":"4fecb6af-5964-4c26-a850-c70c2b30e1bf","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37729","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:11.150464711Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:11.199316301Z","hostname":"buskin","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"3a7d7b8d-a318-40de-8b98-80126248f797","remote_addr":"[fd00:1122:3344:101::b]:61897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48804,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:11.199560338Z","hostname":"buskin","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"a505c359-459b-4caf-b702-4c9e49d7d1b7","remote_addr":"[fd00:1122:3344:101::b]:61897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:13.523308665Z","hostname":"buskin","pid":1618,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:13.523405947Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:13.560982804Z","hostname":"buskin","pid":1618,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999993s","next_cleanup":"Instant { tv_sec: 2215, tv_nsec: 935713439 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:14.642779743Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41449"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:14.649660085Z","hostname":"buskin","pid":1618,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: 07d82d8c-d4f6-4493-b115-dd9d73795a93, project_id: 1dfa68bf-3590-4c60-baac-6934da1753ef }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-03T22:29:14.236318Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(55537))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-63813e9560a\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bdff0a78-722b-4bed-be9c-af91058c9940, read_only_parent: Some(Volume { block_size: 512, id: 012bf74b-f65b-42ae-8fb7-232118cbd3c9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 49fa2a7b-addb-4be2-a55b-0473230b735f, key: Some(\"0bHk2w7E82hUPA4bjI1y5IrKa6UfCw77sBlj+f+jH+w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bdff0a78-722b-4bed-be9c-af91058c9940, key: Some(\"TM4lmqAZS24OBFez5kFIbuFI2LVHmzaNoaxvdz251hc=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAA8p7ODWINYAACns4NYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAPKezg1iDWAAAp7ODWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0NTFkNjE3Yi00ZGQ1LTRjYTYtOWZmZC04Y2E4OTU0ZDBmYTIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTzdUazVXNVk0TklUT0NSNW9XOXQwNDVPNE1veERSMW93VWZyQU5sbC9MciJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:14.649762777Z","hostname":"buskin","pid":1618,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:14.650687523Z","hostname":"buskin","pid":1618,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(55537))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: dd992550-77d6-4fb2-9f40-daf8c9502c8f, kind: Instance { id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 101, 181])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(55537), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-63813e9560a\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bdff0a78-722b-4bed-be9c-af91058c9940, read_only_parent: Some(Volume { block_size: 512, id: 012bf74b-f65b-42ae-8fb7-232118cbd3c9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 49fa2a7b-addb-4be2-a55b-0473230b735f, key: Some(\"0bHk2w7E82hUPA4bjI1y5IrKa6UfCw77sBlj+f+jH+w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bdff0a78-722b-4bed-be9c-af91058c9940, key: Some(\"TM4lmqAZS24OBFez5kFIbuFI2LVHmzaNoaxvdz251hc=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAA8p7ODWINYAACns4NYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAPKezg1iDWAAAp7ODWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0NTFkNjE3Yi00ZGQ1LTRjYTYtOWZmZC04Y2E4OTU0ZDBmYTIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTzdUazVXNVk0TklUT0NSNW9XOXQwNDVPNE1veERSMW93VWZyQU5sbC9MciJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-03T22:29:14.236318Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:14.651314471Z","hostname":"buskin","pid":1618,"uri":"/instances/451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","method":"PUT","req_id":"4a09e40c-cf1e-4f8e-b6cb-79309a28d247","remote_addr":"[fd00:1122:3344:101::a]:41449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8505,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:14.931704044Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:32970"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:14.933676659Z","hostname":"buskin","pid":1618,"uri":"/instances/451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2/state","method":"PUT","req_id":"1400f4a3-9ee1-428d-aa5b-be40b70eb55b","remote_addr":"[fd00:1122:3344:101::a]:32970","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1897,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:14.957171663Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 246, 101, 181]), slot: 0, vni: Vni { inner: 55537 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:14.957939565Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:14.997377842Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:15.022756449Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:16.729337628Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:16.729551214Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 31.449555ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:16.93096702Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 76.255952ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:16.971656613Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 189.800545ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:17.064661671Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 277.827713ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:17.272558078Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 442.133888ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:17.566801078Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 1.458164431s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:18.020864725Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 1.333310822s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:19.488795772Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 1.096774018s","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:20.833110388Z","hostname":"buskin","pid":1618,"zone":"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:21.958267325Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 44.672332ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:21.967502052Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 101.97598ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:22.022839122Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 154.149179ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:22.134216862Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\") failed: Property not found. retry in 268.823419ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:22.297849846Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:22.577405178Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:22.619589081Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAA8p7ODWINYAACns4NYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAPKezg1iDWAAAp7ODWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0NTFkNjE3Yi00ZGQ1LTRjYTYtOWZmZC04Y2E4OTU0ZDBmYTIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTzdUazVXNVk0TklUT0NSNW9XOXQwNDVPNE1veERSMW93VWZyQU5sbC9MciJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-63813e9560a\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: bdff0a78-722b-4bed-be9c-af91058c9940, read_only_parent: Some(Volume { block_size: 512, id: 012bf74b-f65b-42ae-8fb7-232118cbd3c9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 49fa2a7b-addb-4be2-a55b-0473230b735f, key: Some(\"0bHk2w7E82hUPA4bjI1y5IrKa6UfCw77sBlj+f+jH+w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: bdff0a78-722b-4bed-be9c-af91058c9940, key: Some(\"TM4lmqAZS24OBFez5kFIbuFI2LVHmzaNoaxvdz251hc=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 1dfa68bf-3590-4c60-baac-6934da1753ef, silo_id: 07d82d8c-d4f6-4493-b115-dd9d73795a93 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:22.620984231Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:22.743049737Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-04-03T22:29:22.743581132Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:22.743605523Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:22.743612913Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-03T22:29:22.743581132Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }","propolis_id":"58748212-00c9-4674-8645-6c72316d719a"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:22.744124697Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }, propolis_id: 58748212-00c9-4674-8645-6c72316d719a, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-03T22:29:22.743581132Z } }","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-04-03T22:29:22.782690921Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:22.782714192Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:22.782734862Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-03T22:29:22.782690921Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }","propolis_id":"58748212-00c9-4674-8645-6c72316d719a"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:22.782745552Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }, propolis_id: 58748212-00c9-4674-8645-6c72316d719a, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-03T22:29:22.782690921Z } }","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:23.386431615Z","hostname":"buskin","pid":1618,"uri":"/metrics/collect/1ee9e643-97de-413d-8b33-615f41c711a3","method":"GET","req_id":"f0425f7f-60e6-4993-844a-cedf25b4dc4b","remote_addr":"[fd00:1122:3344:101::d]:57652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6053,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.303185512Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59817"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.303801949Z","hostname":"buskin","pid":1618,"uri":"/instances/451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2/state","method":"PUT","req_id":"70a764db-af00-45ea-8b86-2e540a8f7c56","remote_addr":"[fd00:1122:3344:101::a]:59817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":591,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-04-03T22:29:31.303916223Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.303938903Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.303950803Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-03T22:29:31.303916223Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }","propolis_id":"58748212-00c9-4674-8645-6c72316d719a"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.303983744Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(58748212-00c9-4674-8645-6c72316d719a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-03T22:29:14.262283572Z }, propolis_id: 58748212-00c9-4674-8645-6c72316d719a, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-03T22:29:31.303916223Z } }","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-04-03T22:29:31.339211555Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.339232275Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.339249866Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-04-03T22:29:31.339211555Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-03T22:29:31.339211555Z }","propolis_id":"58748212-00c9-4674-8645-6c72316d719a"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.339260756Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.339265986Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.339443181Z","hostname":"buskin","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.339473662Z","hostname":"buskin","pid":1618,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/38c50e2b-3905-4ab7-a650-3ad6b1287295/debug/bundle/zone\", \"/pool/int/939cb7b8-5f2f-4f5c-821a-6bf92b625674/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/7d954909-2e42-44b1-96d9-f536366f6758/crypt/debug\", \"/pool/ext/0c6b0894-9c77-422a-8363-2705812a672c/crypt/debug\", \"/pool/ext/af9b6346-5d28-46df-baea-640dad9b76c3/crypt/debug\", \"/pool/ext/ae4e2e29-ff41-46eb-978b-335ffe5cd37e/crypt/debug\", \"/pool/ext/4eb310e9-0d98-4a07-821c-54f35a32dc4a/crypt/debug\", \"/pool/ext/494a3a1c-f913-4e4b-8eec-202ec6c5bfd4/crypt/debug\", \"/pool/ext/00b9d2d0-59a4-4682-aa18-072c6202c0c8/crypt/debug\", \"/pool/ext/70a16060-4587-476f-9772-16ea3a5a39aa/crypt/debug\", \"/pool/ext/761d3524-c333-4332-9563-7d1ceb36709e/crypt/debug\"] }","zone_name":"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:31.968002246Z","hostname":"buskin","pid":1618,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1236","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a\", bundle_id: 87006862-ded3-4fb6-820d-c5b81449956a }, time_created: 2024-04-03T22:29:31.340275544Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_58748212-00c9-4674-8645-6c72316d719a","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:31.968093458Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:33.767418049Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:33.793184876Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-03T22:29:31.339211555Z }, propolis_id: 58748212-00c9-4674-8645-6c72316d719a, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-03T22:29:33.793117674Z } }","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:33.863156063Z","hostname":"buskin","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59832"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-04-03T22:29:33.864035258Z","hostname":"buskin","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:33.86408924Z","hostname":"buskin","pid":1618,"uri":"/v2p/dd992550-77d6-4fb2-9f40-daf8c9502c8f","method":"DELETE","req_id":"b876ac2a-8929-4265-b510-936401b3a508","remote_addr":"[fd00:1122:3344:101::a]:59832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":745,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:34.232805072Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-03T22:29:31.339211555Z }, propolis_id: 58748212-00c9-4674-8645-6c72316d719a, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-03T22:29:33.793117674Z } }","instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-04-03T22:29:34.236136335Z","hostname":"buskin","pid":1618,"instance_id":"451d617b-4dd5-4ca6-9ffd-8ca8954d0fa2","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"}