[ Apr 29 22:34:29 Enabled. ] [ Apr 29 22:34:29 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Apr 29 22:34:29 Method "start" exited with status 0. ] [ Apr 29 22:34:29 Rereading configuration. ] [ Apr 29 22:34:29 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:29.327839417Z","hostname":"buskin","pid":1590,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:29.327997401Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:29.383181248Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:29.383231979Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:29.501001682Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:29.510165961Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:29.518567462Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.122803806Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.266028439Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.266053619Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.266120201Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.266235403Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.266248584Z","hostname":"buskin","pid":1590,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.266281605Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.266295835Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.269310163Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:30.269320314Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.269347644Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.269359975Z","hostname":"buskin","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.269366995Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.269411256Z","hostname":"buskin","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.269474197Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.269485657Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.269518248Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.281936571Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.281982762Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"dee1c314-6799-4686-a21f-d577311c1a7d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.379609456Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.588366821Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.588402712Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.588407832Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.588423023Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.588429503Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.588447233Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.588453293Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.597039899Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.597067209Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.684047531Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.894840532Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.894885144Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.894920644Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.894926505Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.894955195Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.894988676Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895002036Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895007686Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895035407Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895045717Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895072368Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895076258Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895081308Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895085168Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895100518Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895104518Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895110699Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895125139Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895129259Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895134389Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895150799Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.8951655Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.89516981Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.89517446Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.89519086Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895201141Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895206851Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895211391Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895229271Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895243812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895255352Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895260072Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895281593Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895286143Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895289903Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895294953Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895323303Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895334014Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895338184Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895362804Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895374485Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895435616Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895449196Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895461357Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895472537Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895479807Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895496078Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895507348Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.895516808Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.905933675Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.905967586Z","hostname":"buskin","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999915738s","next_cleanup":"Instant { tv_sec: 1418, tv_nsec: 684820482 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.906040138Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.906077408Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.906083999Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"No ledger in /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.906088389Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909165609Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909258951Z","hostname":"buskin","pid":1590,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909559268Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:979"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909564548Z","hostname":"buskin","pid":1590,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909611019Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.90964835Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.90966Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.90966901Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909689271Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909710641Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909723142Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909729052Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1080"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909732172Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909738142Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909763272Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.909776953Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.91012018Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.910332776Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.910339545Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.910370396Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.910404567Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.910603362Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.910901958Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.910948799Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: ce40678a-3256-41be-b175-a8516c162f00, rack_id: 20971dba-ebe9-4b17-b24a-98610ff87796, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911098033Z","hostname":"buskin","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:58908","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: ce40678a-3256-41be-b175-a8516c162f00, rack_id: 20971dba-ebe9-4b17-b24a-98610ff87796, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911198055Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911204265Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911231186Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911239606Z","hostname":"buskin","pid":1590,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911252726Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911258926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911264957Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911300747Z","hostname":"buskin","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911332308Z","hostname":"buskin","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911339148Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.911342519Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.913538058Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.923769271Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.934075176Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.944119675Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.953597211Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.962043633Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.970397963Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.979344527Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.98868552Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.997167243Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.997226844Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.997232615Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.997239715Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.997246755Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:30.997252645Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.018654343Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.035644079Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.056175887Z","hostname":"buskin","pid":1590,"sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:352"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.056228158Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.056239579Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.056264679Z","hostname":"buskin","pid":1590,"sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:364"} {"msg":"Failed enable ddm stats (will retry after 63.009616ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.059008612Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9a2475d8-a3df-458d-9012-b64c1785093f\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:34:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9a2475d8-a3df-458d-9012-b64c1785093f\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_dee1c314-6799-4686-a21f-d577311c1a7d/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.071547357Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_dee1c314-6799-4686-a21f-d577311c1a7d/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.094882329Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_dee1c314-6799-4686-a21f-d577311c1a7d/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.09493281Z","hostname":"buskin","pid":1590,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.099896103Z","hostname":"buskin","pid":1590,"sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:379"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.099914134Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_dee1c314-6799-4686-a21f-d577311c1a7d/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.099928854Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.123139643Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 176.466441ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.123673005Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6256b11e-8446-463b-b10f-eb2d3427c401\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:34:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6256b11e-8446-463b-b10f-eb2d3427c401\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.169355555Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.300433031Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 196.313024ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.301092936Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1bf4fd95-0828-4a70-9a72-e949a417af39\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:34:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1bf4fd95-0828-4a70-9a72-e949a417af39\" }"} {"msg":"Mounting oxi_dee1c314-6799-4686-a21f-d577311c1a7d/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.388080018Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.399978749Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.400064001Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.400113652Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.400130602Z","hostname":"buskin","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.499079516Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 99.660285ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.49971027Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ec25c0fe-9cab-4e69-a5fc-4757a48f6d28\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:34:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ec25c0fe-9cab-4e69-a5fc-4757a48f6d28\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.601025998Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 185.417224ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.601617612Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d55f8cf1-9e82-4418-8b36-383ece3d50e1\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:34:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d55f8cf1-9e82-4418-8b36-383ece3d50e1\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.787838044Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 214.108926ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:31.788583591Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"59c84fc9-b8bd-4ebb-8f57-03df69fd3230\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:34:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"59c84fc9-b8bd-4ebb-8f57-03df69fd3230\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.003516157Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 356.008912ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.004338725Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"30ee60ab-e520-4861-b816-5541e2577b54\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:34:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"30ee60ab-e520-4861-b816-5541e2577b54\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.106407681Z","hostname":"buskin","pid":1590,"sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.107017774Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.107045455Z","hostname":"buskin","pid":1590,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.107125987Z","hostname":"buskin","pid":1590,"file":"oximeter/producer/src/lib.rs:284","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:284","interval":"30s","address":"[fd00:1122:3344:101::1]:46822","producer_id":"ce40678a-3256-41be-b175-a8516c162f00","route":"//ce40678a-3256-41be-b175-a8516c162f00"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:32.113633715Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"287.354429ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.155734804Z","hostname":"buskin","pid":1590,"component":"VmmReservoirManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.155778035Z","hostname":"buskin","pid":1590,"component":"VmmReservoirManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.155921018Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.155931578Z","hostname":"buskin","pid":1590,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.156796128Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.156823819Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.15685115Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.15686475Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.157120486Z","hostname":"buskin","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.15732245Z","hostname":"buskin","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.157352021Z","hostname":"buskin","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.157371881Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.157377542Z","hostname":"buskin","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:58908","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.157414632Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.157423183Z","hostname":"buskin","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.325907941Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.325972682Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.326004603Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.326023613Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.326041333Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:32.330644348Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.357764756Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:59177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.35792205Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.358021512Z","hostname":"buskin","pid":1590,"uri":"/inventory","method":"GET","req_id":"7e89b6f3-7170-431e-b0b8-5736429cd87f","remote_addr":"[fd00:1122:3344:101::1]:59177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":166,"response_code":"200"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.361685256Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.387177766Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.387294999Z","hostname":"buskin","pid":1590,"uri":"/sled-role","method":"GET","req_id":"771e75bc-fd86-4574-b777-6b321d23034d","remote_addr":"[fd00:1122:3344:101::1]:35116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":72,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.387957604Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.387980124Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.388260591Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.388537917Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:32.402209628Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"371.414416ms"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 8b6d3c76-b4fe-4c73-a771-52e4601b4e70, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool) }, OmicronPhysicalDiskConfig { id: f3427a36-bf1f-4472-8020-41a507bb3fb5, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool) }, OmicronPhysicalDiskConfig { id: e35db565-2d4c-41c0-8105-825da83614ea, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool) }, OmicronPhysicalDiskConfig { id: 82d05f31-748e-447b-9d96-2c55fe306b65, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool) }, OmicronPhysicalDiskConfig { id: e0942b23-3d88-4b57-9571-e0eb086622b2, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool) }, OmicronPhysicalDiskConfig { id: 23ee0821-5890-4e62-ac9c-c9440a9e67cc, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool) }, OmicronPhysicalDiskConfig { id: 6cdf5a7c-fb2e-48fa-8f89-3e54df6b016a, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: ff581678-2bea-470b-9448-890977b6d596 (zpool) }, OmicronPhysicalDiskConfig { id: f34e6f21-391b-4b8c-97cd-bc0a68a7a135, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool) }, OmicronPhysicalDiskConfig { id: db270124-36f7-4234-838a-35bfd875db78, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.426313998Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:346"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.426465541Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33800"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: 8b6d3c76-b4fe-4c73-a771-52e4601b4e70, pool_id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: f3427a36-bf1f-4472-8020-41a507bb3fb5, pool_id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: e35db565-2d4c-41c0-8105-825da83614ea, pool_id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: 82d05f31-748e-447b-9d96-2c55fe306b65, pool_id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: e0942b23-3d88-4b57-9571-e0eb086622b2, pool_id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: 23ee0821-5890-4e62-ac9c-c9440a9e67cc, pool_id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: 6cdf5a7c-fb2e-48fa-8f89-3e54df6b016a, pool_id: ff581678-2bea-470b-9448-890977b6d596 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: f34e6f21-391b-4b8c-97cd-bc0a68a7a135, pool_id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: db270124-36f7-4234-838a-35bfd875db78, pool_id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.426727067Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.426738637Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.426753328Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.426758558Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.426810089Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.426815769Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.426821179Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.426825669Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.524601736Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:32.524651068Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"9b351946-585e-4029-8f1b-9f69550dce71 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:32.887140465Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.49322546s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:34:33.305646628Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:33.491528162Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:33.660709496Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:33.660807848Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:33.660820178Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:33.660830839Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:33.660843819Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:33.660964892Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:33.660975912Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:34.254088243Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:34.256668991Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:34:34.457827604Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:34.457937416Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.482569533s"} {"msg":"Automatically destroying dataset: oxp_9b351946-585e-4029-8f1b-9f69550dce71/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:34.463175755Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:34.789196642Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 67.210545ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:35.521187266Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 64.407464ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:35.923757576Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:34:36.135239723Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 136.218312ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:36.350002175Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:36.351283205Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:37.062728461Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.856364358s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 282.374829ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:37.062900454Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:34:37.297409876Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_9b351946-585e-4029-8f1b-9f69550dce71","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:37.768095658Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 997.917983ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:37.98810712Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:37.995611811Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:37.995623051Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:37.995659252Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:37.995665532Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:37.995670852Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:38.207453046Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:38.207505147Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:39.37494296Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:39.594631155Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:39.594680716Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:39.594691786Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:39.594708886Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:39.594754948Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:39.594900741Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:39.594907711Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:34:40.321806689Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:40.32184914Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 562.295679ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:40.321991643Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:40.552720439Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_478a6e43-eea9-40ca-b6f4-6a492473e177/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:40.808005744Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.43473497Z","hostname":"buskin","pid":1590,"component":"VmmReservoirManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:42.436626153Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.49906963s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:34:42.436913689Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:42.436944Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_478a6e43-eea9-40ca-b6f4-6a492473e177","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.522675073Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.531216708Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.531261238Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.531268949Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.531274179Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.531280169Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.541639305Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.541685526Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"e3118907-da30-4e85-a767-c5a42a2e46ca (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.634110111Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.642025672Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.642072663Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.642089463Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.642100273Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.642112763Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.642248207Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.642255507Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.679425453Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e3118907-da30-4e85-a767-c5a42a2e46ca/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.686975385Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_e3118907-da30-4e85-a767-c5a42a2e46ca","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.809194349Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.817636002Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.817678273Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.817686023Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.817693063Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.817697493Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.82807362Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.828116501Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"d280116a-a989-463f-b7e1-373c7fcb5463 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.921837495Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.929640493Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.929685964Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.929693445Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.929737246Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.929748956Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.92991171Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.929918899Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.966455712Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d280116a-a989-463f-b7e1-373c7fcb5463/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:42.974633818Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_d280116a-a989-463f-b7e1-373c7fcb5463","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.100209868Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.10817441Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.108226551Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.108232891Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.108238011Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.108242331Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.118255149Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.11829056Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.211974254Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.220273443Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.220313154Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.220320324Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.220368345Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.220379676Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.22055793Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.2205655Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.256187081Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_661f9fd6-5e25-45d3-955c-06a9af9dad35/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.26403437Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_661f9fd6-5e25-45d3-955c-06a9af9dad35","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.383769008Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.391519264Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.391566805Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.391593286Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.391600136Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.391605166Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.401480291Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.401511712Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.495681787Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.504006036Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.504035107Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.504042397Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.504061658Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.504069648Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.504192001Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.504198651Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:34:43.508218542Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.539197038Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9962603d-6bc6-41a3-a346-9382c9e6bfb4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.547018196Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_9962603d-6bc6-41a3-a346-9382c9e6bfb4","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.6705303Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.678222875Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.678254336Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.678282716Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.678290366Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.678295017Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.688737635Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.688761965Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ff581678-2bea-470b-9448-890977b6d596 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.78204334Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.789832407Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.789862948Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.789894989Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.789923679Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.789930609Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.790073813Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.790083863Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.82549839Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ff581678-2bea-470b-9448-890977b6d596/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.833679346Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_ff581678-2bea-470b-9448-890977b6d596","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.957596959Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.965423387Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.965450318Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.965455038Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.965460538Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.965464468Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.974963004Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:43.974980855Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"4b1821a9-dc17-4642-9053-2192b3ae698e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.067767258Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.075622267Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.075650868Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.075672349Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.075678179Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.075690299Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.075875433Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.075891534Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:44.079959516Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.113160832Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4b1821a9-dc17-4642-9053-2192b3ae698e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.121454951Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_4b1821a9-dc17-4642-9053-2192b3ae698e","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.242077029Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.249721693Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.249765774Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.249772484Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.249788555Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.249794175Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.258950753Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.258976524Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"3c2c8854-3269-4025-ae74-138b796a9368 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.351685816Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.359498804Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.359535755Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.359547485Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.359563445Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.359572796Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.359688438Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.359695378Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.385972367Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.395002963Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3c2c8854-3269-4025-ae74-138b796a9368/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.402747499Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:34:44.510552525Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_3c2c8854-3269-4025-ae74-138b796a9368","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.528282539Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.536059776Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.536104117Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.536125247Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.536293171Z","hostname":"buskin","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dee1c314-6799-4686-a21f-d577311c1a7d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 2a8df99a-fa8f-45a4-9b0b-dc55770b6483 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.536496776Z","hostname":"buskin","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.536804193Z","hostname":"buskin","pid":1590,"uri":"/omicron-physical-disks","method":"PUT","req_id":"f00533be-e828-4f82-9e44-ab6f0f4e0d78","remote_addr":"[fd00:1122:3344:101::1]:33800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12110258,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.567653245Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.567804339Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:41025"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.568077175Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.568082295Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.617623934Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.617659075Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.666469687Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.666510057Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.714265845Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.714339237Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.714357717Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.714454429Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.760186191Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.760240652Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.760246883Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.809661998Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.813298501Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.817049797Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.829664414Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.834083425Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:44.838182548Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:34:45.512649702Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:46.078065151Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:34:46.5148164Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:47.055207419Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:47.055279441Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:47.055469715Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:47.538798005Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:48.020223941Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:48.020324133Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:48.020332874Z","hostname":"buskin","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899\") failed: Property not found. retry in 29.874424ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.028628563Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:48.028663303Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:48.032238465Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899\") failed: Property not found. retry in 74.883215ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.067284683Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899\") failed: Property not found. retry in 119.12835ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.151461841Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f\") failed: Property not found. retry in 71.936409ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.158037661Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50\") failed: Property not found. retry in 28.212338ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.164003466Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50\") failed: Property not found. retry in 69.694794ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.199401413Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f\") failed: Property not found. retry in 95.394853ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.406863789Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899\") failed: Property not found. retry in 215.53552ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.444374753Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50\") failed: Property not found. retry in 186.831274ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.452944688Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f\") failed: Property not found. retry in 299.243987ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.529734377Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50\") failed: Property not found. retry in 375.121019ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.654324686Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899\") failed: Property not found. retry in 982.644419ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.662191365Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f\") failed: Property not found. retry in 326.693515ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:48.842335728Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50\") failed: Property not found. retry in 851.703086ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:49.038871805Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f\") failed: Property not found. retry in 592.860962ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:49.17822652Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899\") failed: Property not found. retry in 1.457481383s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:49.654403127Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f\") failed: Property not found. retry in 785.581953ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:49.780794876Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50\") failed: Property not found. retry in 655.430838ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:49.901318891Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:50.436243656Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:50.436268456Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50\") failed: Property not found. retry in 1.283623789s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:50.564574959Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f\") failed: Property not found. retry in 1.361423777s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:50.57470616Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe0c:d2e8","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:50.981081096Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:50.986326866Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:51.104930778Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"} {"msg":"wait for service Some(\"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899\") failed: Property not found. retry in 714.719956ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:51.121061605Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:51.352524538Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:51.538611627Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:51.830313101Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"} {"msg":"wait for service Some(\"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899\") failed: Property not found. retry in 906.866084ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:51.844873763Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50\") failed: Property not found. retry in 1.442996403s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:51.857157463Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f\") failed: Property not found. retry in 541.799184ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:51.943989011Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:52.040315455Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:52.227634662Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:52.453360204Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"} {"msg":"wait for service Some(\"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f\") failed: Property not found. retry in 1.173520267s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:52.495379461Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:52.645509921Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:52.656253176Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:52.656313297Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:52.656326927Z","hostname":"buskin","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899\") failed: Property not found. retry in 550.367761ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:52.76137571Z","hostname":"buskin","pid":1590,"zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.213321175Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.220855587Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.388501106Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.503056435Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.524752559Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.626002176Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.688802056Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_a0ff8ea7-90b6-41b5-8781-fe3c2645f899"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.688862517Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_bcb9494f-42ec-4d91-a912-48c9b354aa2f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.688867728Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.688876318Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.689202455Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.689474002Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"7b63415c-59b7-4080-a766-7a74951a3f43","remote_addr":"[fd00:1122:3344:101::1]:41025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9121593,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.691127279Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.752984188Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.785032288Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.816166647Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:589"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.816242459Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:627"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.816616888Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:630","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.816626238Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892110017Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892182629Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892192459Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892199159Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892204929Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.89221147Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.89222785Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.89223409Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.89223971Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892246981Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892254251Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892260041Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892265801Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892271611Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892277651Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892283121Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892288821Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892295851Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892313622Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892319472Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892323712Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892327832Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.892331912Z","hostname":"buskin","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"failed to write DNS configuration (will retry in 59.687241ms)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:53.903271812Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:567","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:53.981548344Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.032930935Z","hostname":"buskin","pid":1590,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.078197846Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:583"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.078210676Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.110676536Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.111395812Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.111401052Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.140158957Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.14027735Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58152"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.141378475Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.141388435Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.161583765Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.161624796Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.181461928Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.181509599Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.198976017Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.199045439Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.199277274Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.199418287Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.211163485Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.243332078Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.244101595Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.244113855Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.289286134Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 173, 41]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.289334545Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.289927179Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.289937069Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.290258946Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.2926263Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.316081915Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:54.333693326Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:55.466343597Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:55.46649644Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a\") failed: Property not found. retry in 68.390558ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:55.603986712Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a\") failed: Property not found. retry in 52.295121ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:55.684353123Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a\") failed: Property not found. retry in 206.803451ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:55.760926267Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a\") failed: Property not found. retry in 341.017506ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:55.978674427Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a\") failed: Property not found. retry in 1.054915944s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:56.328428914Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Updated Peer Addresses: {[fdb0:edc4:a262:183c::1]:12347}","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:57.060996491Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:514"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:57.119601616Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:34:57.1197557Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"2d959eb5-eb1d-4de1-8db6-6519ba91ce9e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59900","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":80,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a\") failed: Property not found. retry in 696.000719ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:57.394545189Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a\") failed: Property not found. retry in 800.661455ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:58.101207966Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a\") failed: Property not found. retry in 1.074722921s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:58.910850029Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a\") failed: Property not found. retry in 1.041247176s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:34:59.995809963Z","hostname":"buskin","pid":1590,"zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.056575266Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.056660318Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.056991866Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.057299623Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b4f9d794-0608-4112-acc2-2b40f94d0ee5","remote_addr":"[fd00:1122:3344:101::1]:58152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6916905,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.059020422Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:624"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.088042203Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.088235847Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51525"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.097064468Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.097278573Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"8a3fc52d-6edc-4d05-a805-4f0bd7e2ed9b","remote_addr":"[fd00:1122:3344:101::1]:51525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8935,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:01.099465113Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.250987445Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:01.251114898Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33569"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:05.259360842Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:05.259541136Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"8ffcc459-348a-4f92-b3ab-7b83388ff863","remote_addr":"[fd00:1122:3344:101::1]:33569","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008272,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:05.262701218Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:05.451685303Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:05.451816886Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38754"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:05.461342613Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:05.461492016Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"a3fffc7c-9043-4395-b19f-d90f8c11d3c5","remote_addr":"[fd00:1122:3344:101::1]:38754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9529,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:05.464682039Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:05.578764428Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:05.57887122Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63764"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_903782fb-f735-42e4-99f5-7de3123abd5a': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:09.58828401Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:09.588384643Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"46fa8ac1-2673-4d48-a13e-48ccb447a516","remote_addr":"[fd00:1122:3344:101::1]:63764","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009418,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:09.591311809Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:09.880682681Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:09.880798284Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:59940"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:09.888793086Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"95ae2b87-baac-4fff-a353-c4feee1e734c","remote_addr":"[fd00:1122:3344:101::1]:59940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7844,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714430105.324157, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:09.890257569Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:09.890261949Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.123568034Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.123689466Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47213"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.131893283Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"0a7493c1-1546-4f47-a82c-3447e4c4e86e","remote_addr":"[fd00:1122:3344:101::1]:47213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8079,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714430105.324157, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.133544911Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:10.133551991Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.315353402Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.315461315Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51275"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.32404654Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"cb0ba14e-8eec-4709-a306-15b53e8c19a8","remote_addr":"[fd00:1122:3344:101::1]:51275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8460,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714430105.324157, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.325646267Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:10.325652417Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.567894265Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.567985207Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52705"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.57602682Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"2783f752-08e0-4faa-b264-7d722f135b71","remote_addr":"[fd00:1122:3344:101::1]:52705","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7881,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714430105.324157, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:10.577466613Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:10.577470823Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:11.062535022Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:11.062631874Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50133"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:11.071853895Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"9a0dcafd-7967-4682-bcdb-1b9f9b65f365","remote_addr":"[fd00:1122:3344:101::1]:50133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9090,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714430105.324157, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:11.073243586Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:11.073249706Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:11.703497273Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:11.703599035Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58766"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:11.711184908Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"921158e4-813e-40a6-a3dd-268ad1aadea4","remote_addr":"[fd00:1122:3344:101::1]:58766","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7450,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714430105.324157, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:11.712701582Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:11.712707103Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:12.366226269Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:12.366385943Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:12.373941525Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"e95610db-cb03-4e9b-b656-480a9a6117c6","remote_addr":"[fd00:1122:3344:101::1]:48808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7518,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714430105.324157, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:12.377435954Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:12.377445475Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:12.947615632Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:12.947710355Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:57478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:12.954815156Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"5cf5f18b-c130-45a1-bea9-e2fce1576c21","remote_addr":"[fd00:1122:3344:101::1]:57478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6974,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714430105.324157, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:12.956206308Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:12.956212778Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:12.974501985Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"259.064052ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.445815346Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.445910818Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39530"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.449139651Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.466317762Z","hostname":"buskin","pid":1590,"uri":"/timesync","method":"GET","req_id":"7b74473d-a526-4ca1-b596-8a945e60852d","remote_addr":"[fd00:1122:3344:101::1]:39530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":20268,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 94448712, ip_addr: 5.161.44.72, stratum: 3, ref_time: 1714430112.8242695, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.468457471Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.468464551Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1192"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.497196976Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.49738067Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50259"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.497565794Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.497573554Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.515655897Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.515681487Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.533294528Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.533320339Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.551408941Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.551438352Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.602648548Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.602678679Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.656011684Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.656036354Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.707466546Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.707493156Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.757917205Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.757946176Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.80948712Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.809539921Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.809755566Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.809896499Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:13.844412645Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:13.844483307Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:13.844494077Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:13.844503317Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:13.844512128Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.84461129Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"00ece362-9b13-4fa4-9a40-15c7347514cf","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized)]","latency_us":347160,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:13.844885406Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"96.474812ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"00ece362-9b13-4fa4-9a40-15c7347514cf\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"00ece362-9b13-4fa4-9a40-15c7347514cf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.943046762Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.943440301Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.943451071Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.962822252Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.962864073Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.980412733Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.980465004Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.998770221Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:13.998797632Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.017858736Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.017889547Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.036204544Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.036235835Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.054228845Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.054250725Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.071975599Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.071997859Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.089798225Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.089819065Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.089998699Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.090112182Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.121544028Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.121592429Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.121600439Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.12163053Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.12163863Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.121732132Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"ddbe6fd7-c937-4dcd-905b-c4e2ba103a3d","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized)]","latency_us":178449,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.121881155Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"151.737742ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ddbe6fd7-c937-4dcd-905b-c4e2ba103a3d\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ddbe6fd7-c937-4dcd-905b-c4e2ba103a3d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.275043584Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.275460744Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.275475344Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.295671034Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.295720426Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.314100034Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.314133975Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.331544491Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.331578942Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.35033257Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.35036717Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.368961694Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.368993535Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.387478726Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.387512846Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.405776453Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.405821954Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.424055829Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.42411282Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.424286314Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.424428017Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.457504841Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.457559852Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.457569952Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.457577842Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.457585873Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.457720086Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"23199666-b03d-409e-b701-0e55b9c91da9","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized)]","latency_us":182473,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.45790988Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"211.24851ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"23199666-b03d-409e-b701-0e55b9c91da9\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"23199666-b03d-409e-b701-0e55b9c91da9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.669827647Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.670954973Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.670979284Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.690153Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.690194111Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.708072159Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.70811012Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.727073201Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.727107962Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.747025946Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.747080957Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.764697209Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.764734939Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.783095448Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.783134338Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.801124728Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.801157889Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.818840882Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.818879933Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.819081337Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.81919002Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.852372906Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.852436567Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.852446967Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.852455077Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.852464038Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.85256811Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"3b4228dd-c98b-4fbc-846d-3059bb3ea91c","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized)]","latency_us":182550,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:14.852738424Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"119.74638ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3b4228dd-c98b-4fbc-846d-3059bb3ea91c\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3b4228dd-c98b-4fbc-846d-3059bb3ea91c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.973902224Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.974262222Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.974276412Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.993036739Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:14.993081341Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.01105676Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.011102991Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.028705482Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.028755743Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.046848075Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.046889166Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.065316966Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.065359377Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.083840098Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.083895279Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.102204276Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.102247937Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.120953433Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.121012385Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.12123725Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.121392303Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.155407828Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.15546223Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.15547232Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.15548145Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.1554884Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.155646704Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"888eb586-97e2-44dc-a531-e345b9057dbd","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized)]","latency_us":181582,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.155857979Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"298.531603ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"888eb586-97e2-44dc-a531-e345b9057dbd\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"888eb586-97e2-44dc-a531-e345b9057dbd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.456270592Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.457483809Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.45749075Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.477039425Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.477070445Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.495770102Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.495805292Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.513940125Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.513983926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.531778162Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.531805812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.549364492Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.549391073Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.567191998Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.567219599Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.58569721Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.585726441Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.603460664Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.603499865Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.60367734Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.603801132Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.635690179Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.6357338Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.63574048Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.63574945Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.63575772Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.635833822Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"01e03f84-dc2d-4240-a4ad-f314c20300cc","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized)]","latency_us":179348,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:15.635945884Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"315.136283ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"01e03f84-dc2d-4240-a4ad-f314c20300cc\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"01e03f84-dc2d-4240-a4ad-f314c20300cc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.952369032Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.952768712Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.952786062Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.972705615Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.972742616Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.991259578Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:15.991289989Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.009397381Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.009448812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.029261184Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.029299144Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.048270587Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.048309618Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.066152224Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.066186525Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.085508895Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.085551066Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.104564839Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.10461754Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.104848745Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.105003879Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.13755477Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.137611762Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.137621772Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.137652093Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.137662853Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.137800986Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"590606ba-e409-42d4-a866-975f961d5ee2","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized)]","latency_us":185234,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.13796198Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"414.682589ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"590606ba-e409-42d4-a866-975f961d5ee2\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"590606ba-e409-42d4-a866-975f961d5ee2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.554387986Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.554765844Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.554786165Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.574568145Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.574615957Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.593773323Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.593828014Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.612478689Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.61252463Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.631182155Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.631232236Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.65116811Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.651201361Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.670120352Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.670154873Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.688878299Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.68893115Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.707403591Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.707466153Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.707689878Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.707844531Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.741307594Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.741362745Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.741372985Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.741382915Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.741393276Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:16.741502678Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1bd1cf34-3520-4a3c-972f-99f778ed8e22","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized)]","latency_us":186938,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:16.741669872Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"478.349194ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1bd1cf34-3520-4a3c-972f-99f778ed8e22\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1bd1cf34-3520-4a3c-972f-99f778ed8e22\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.221076552Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.221457991Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.221470991Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.240593247Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.240642348Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.258352801Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.258394832Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.277935277Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.277985928Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.296615713Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.296662624Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.31492108Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.314961631Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.332724195Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.332767566Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.350770237Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.350812287Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.369686797Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.369738929Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.369906912Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.370038305Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.401455861Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.401499782Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.401506512Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.401514182Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.401520313Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.401633425Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"cc38fe8c-1eef-42ce-a2c3-1ff589ded2d4","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized)]","latency_us":180369,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.40184036Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"327.136286ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cc38fe8c-1eef-42ce-a2c3-1ff589ded2d4\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cc38fe8c-1eef-42ce-a2c3-1ff589ded2d4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.730277211Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.730627759Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.7306813Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.750930762Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.750964393Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.768694826Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.768716607Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.785598991Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.785623602Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.80310779Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.803138981Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.821394407Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.821421838Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.839638553Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.839671103Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.85751654Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.85753805Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.877812382Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.877862433Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.878043667Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.87816163Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.91020748Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.910255681Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.910265641Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.910275802Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.910283362Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:17.910368824Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"0c8190fb-ec57-4b97-90ac-11135888485f","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized)]","latency_us":179929,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:17.910543438Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"521.990506ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0c8190fb-ec57-4b97-90ac-11135888485f\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0c8190fb-ec57-4b97-90ac-11135888485f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.433975781Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.434351179Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.43438582Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.452918322Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.452953473Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.470063893Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.470104864Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.48659943Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.4866443Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.505545251Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.505583002Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.523109571Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.523148332Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.540853806Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.540888466Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.558825625Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.558860326Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.576788894Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.576847935Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.577001529Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.577113592Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:18.609646422Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:18.609689783Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:18.609697704Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:18.609705264Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:18.609714384Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:18.609879388Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"2b7f02d8-5d02-421e-9d08-627c92ef8754","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized)]","latency_us":175718,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:18.610072782Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"653.12255ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2b7f02d8-5d02-421e-9d08-627c92ef8754\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2b7f02d8-5d02-421e-9d08-627c92ef8754\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.263542197Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.264666883Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.264674753Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.284054755Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.284121946Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.303041647Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.303101598Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.32028863Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.320337731Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.338877063Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.338935165Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.357635351Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.357693152Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.375479737Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.375543039Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.392957115Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.393015087Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.411866436Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.411934348Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.412181343Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.412301536Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:19.445348439Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:19.44539349Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:19.44540146Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:19.445425791Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:19.445432621Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:19.445551483Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"8dbb79f7-2f23-4e56-844e-b190ee77f088","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized)]","latency_us":181797,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:19.445754268Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"880.998286ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8dbb79f7-2f23-4e56-844e-b190ee77f088\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8dbb79f7-2f23-4e56-844e-b190ee77f088\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.328318692Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.32864143Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.32865447Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.348924812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.348973062Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.368412125Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.368438906Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.386300743Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.386330253Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.404135339Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.40416909Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.423322096Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.423347827Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.44148442Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.44151149Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.459875819Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.45991537Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.47748127Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.477522041Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.477722145Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.477848848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:20.510567934Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:20.510603214Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:20.510610044Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:20.510618795Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:20.510625725Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:20.510743438Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e88b1a7e-9eaa-48eb-887e-83564accb908","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized)]","latency_us":182278,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:20.510944082Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"690.456524ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e88b1a7e-9eaa-48eb-887e-83564accb908\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e88b1a7e-9eaa-48eb-887e-83564accb908\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.203426836Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.203822995Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.203897807Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.223268608Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.223304949Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.241137195Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.241175906Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.258707585Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.258747136Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.27647226Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.276523521Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.294511491Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.294552632Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.312061471Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.312099231Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.330557762Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.330597723Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.348283586Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.348331097Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.348530921Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.348667534Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:21.379791513Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:21.379856685Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:21.379867985Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:21.379878275Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:21.379886385Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:21.380044459Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"8b173ba9-6e0c-47ef-9e9c-1210d5ca3055","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized)]","latency_us":176414,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:21.380237304Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"708.605666ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8b173ba9-6e0c-47ef-9e9c-1210d5ca3055\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8b173ba9-6e0c-47ef-9e9c-1210d5ca3055\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.089757226Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.090171655Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.090186095Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.110813655Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.110867456Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.129293506Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.129334747Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.1470433Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.147090752Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.166106025Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.166157046Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.184089424Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.184135045Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.202148456Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.202206727Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.220224518Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.220276059Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.238238058Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.238293819Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.238455633Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.238572896Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:22.271855934Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:22.271891655Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:22.271897955Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:22.271903065Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:22.271908205Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:22.272022778Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"bf0f349a-c16e-4fff-8723-4cde6e3e6155","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized)]","latency_us":182085,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:22.272182931Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.704101948s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bf0f349a-c16e-4fff-8723-4cde6e3e6155\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bf0f349a-c16e-4fff-8723-4cde6e3e6155\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:23.976690548Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:23.977050116Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:23.977060237Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:23.99741235Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:23.997454281Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.01453503Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.014569631Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.031796413Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.031836544Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.050784326Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.050820477Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.068728114Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.068755665Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.087658276Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.087694467Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.105316338Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.105343069Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.123706767Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.123765728Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.123979053Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.124105876Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:24.155993142Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:24.156063214Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:24.156075894Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:24.156084584Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:24.156092395Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:24.156204597Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"cdf9de82-22a1-4d8d-ba76-cb02bd3e8b30","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized)]","latency_us":179346,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:24.156399292Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.068504926s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cdf9de82-22a1-4d8d-ba76-cb02bd3e8b30\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cdf9de82-22a1-4d8d-ba76-cb02bd3e8b30\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.225589246Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.225978305Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.225990545Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.246464112Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.246497662Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.265527626Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.265563876Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.284188311Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.284231562Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.302770044Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.302799395Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.321713895Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.321767287Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.339394538Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.339427269Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.358873152Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.358909753Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.378233823Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.378266814Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.378463728Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.378589651Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:26.411722976Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:26.411764227Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:26.411769327Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:26.411776257Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:26.411797918Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:26.41189393Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"39ca86f0-0d5c-4630-9d60-fda5787d4a9e","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized)]","latency_us":186102,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:26.412080294Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.537451394s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"39ca86f0-0d5c-4630-9d60-fda5787d4a9e\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"39ca86f0-0d5c-4630-9d60-fda5787d4a9e\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:27.138485571Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36656"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:27.138595143Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"4d21a73c-7ffe-40c8-b55f-01f65bc2e64f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36656","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":53,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:27.950987979Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:27.951367168Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:27.951382858Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:27.971287511Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:27.971332322Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:27.991549963Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:27.991584753Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.009453311Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.009490741Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.028908134Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.028947865Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.047467097Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.047499957Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.065780404Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.065817664Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.084738655Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.084776986Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.10205077Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.102106861Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.102318956Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.102467779Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:28.134220113Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:28.134268464Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:28.134278454Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:28.134288314Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:28.134298235Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:28.134436318Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"8296fa23-97d6-4a25-a177-0497e7fa450b","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized)]","latency_us":183276,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:28.134591251Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.949055752s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8296fa23-97d6-4a25-a177-0497e7fa450b\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8296fa23-97d6-4a25-a177-0497e7fa450b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:28.251853262Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"724.964606ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.084774203Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.08508754Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.085096321Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.10660367Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.106652062Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.125283946Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.125328217Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.144819871Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.144860372Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.163546648Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.163584899Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.18211109Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.182162232Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.201661776Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.201714427Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.219924492Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.219971663Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.23871381Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.238762271Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.238966556Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.239103879Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:31.27163013Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:31.271661901Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:31.271668291Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:31.271673751Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:31.271680081Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:31.271793744Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"8d0c7501-d27f-4deb-9e78-3bd1aad1107a","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized)]","latency_us":186895,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:31.271967728Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.560147274s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8d0c7501-d27f-4deb-9e78-3bd1aad1107a\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8d0c7501-d27f-4deb-9e78-3bd1aad1107a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.832974793Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.834069449Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.834082749Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.854547245Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.854606666Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.873354413Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.873383934Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.88901788Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.889053701Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.908100135Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.908139796Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.925683525Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.925715426Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.943591713Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.943624044Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.96187782Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.96190467Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.978094649Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.9781351Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.978349705Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:34.978479208Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:35.009707309Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:35.009778041Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:35.009787811Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:35.009796721Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:35.009805241Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:35.009913974Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"eee2faf0-3dc1-4c18-ab86-57aadfe625d1","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized)]","latency_us":176720,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:35.010105028Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.400196388s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eee2faf0-3dc1-4c18-ab86-57aadfe625d1\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eee2faf0-3dc1-4c18-ab86-57aadfe625d1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.411259184Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.411657033Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.411697054Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.431586577Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.431613358Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.45015468Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.450185081Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.469098162Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.469126272Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.487700485Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.487735156Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.506036193Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.506065104Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.524110885Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.524133985Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.540388556Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.540412856Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.558687393Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.558738974Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.558942198Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.559043321Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:37.59019465Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:37.590267382Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:37.590277312Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:37.590287952Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:37.590298082Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:37.590415905Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e74519d8-24c0-4726-8da6-2e57345fcf65","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized)]","latency_us":178962,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:37.59063377Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.554053602s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e74519d8-24c0-4726-8da6-2e57345fcf65\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e74519d8-24c0-4726-8da6-2e57345fcf65\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.146278664Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.146663153Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.146683204Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.167071508Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.167127019Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.185129289Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.185190791Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.203276923Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.203329434Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.221845356Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.221892277Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.24001912Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.240060791Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.257741293Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.257780854Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.276567732Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.276609313Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.294875849Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.294936171Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.295154016Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.295282278Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:41.32825941Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:41.328303301Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:41.328310841Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:41.328317731Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:41.328326781Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:41.328497395Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"d4d7432c-0249-4787-b795-99ca981c0659","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized)]","latency_us":182056,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:41.32872557Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.288435713s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d4d7432c-0249-4787-b795-99ca981c0659\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d4d7432c-0249-4787-b795-99ca981c0659\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:44.011983222Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.209157178s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.617974173Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.618325641Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.618356192Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.638939511Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.638991772Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.657929103Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.657973914Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.675605636Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.675644967Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.693076434Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.693115965Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.712123828Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.712166879Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.729507904Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.729546095Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.74735677Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.747403071Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.765687858Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.765735109Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.765970274Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.766112698Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:47.797794519Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:47.79782853Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:47.797849921Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:47.797856741Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:47.797864381Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:47.797975003Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"6a15c36d-2e71-4260-bd16-f2b6b3496a99","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized)]","latency_us":179825,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:47.798179808Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.476023109s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a15c36d-2e71-4260-bd16-f2b6b3496a99\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6a15c36d-2e71-4260-bd16-f2b6b3496a99\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.275166507Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.275537175Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.275553156Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.297509446Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.297550697Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.316381846Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.316423067Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.335380999Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.33541355Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.354105235Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.354134966Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.371977522Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.372009413Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.390373371Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.390404462Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.408566866Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.408602127Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.427216021Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.427256131Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.427471587Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.427587329Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:54.459810313Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:54.459858624Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:54.459867265Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:54.459876995Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:54.459884725Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:54.460057689Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1992d377-5d8b-45e6-be79-626006e0ad4e","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized)]","latency_us":184730,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:35:54.460307815Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"5.461632928s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1992d377-5d8b-45e6-be79-626006e0ad4e\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1992d377-5d8b-45e6-be79-626006e0ad4e\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:57.173321904Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52960"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:57.173424637Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"4f1f1b01-1d28-4a62-ac1a-400510efbd1a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52960","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":60,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:59.922671862Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:59.923961181Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:59.923971161Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:59.943678401Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:59.943727472Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:59.961848504Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:59.961901095Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:59.98187256Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:35:59.981951652Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.00205585Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.002103641Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.022071556Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.022117197Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.041225262Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.041277133Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.059391216Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.059436467Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.07798949Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.078052621Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.078285116Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.07842644Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:00.111002442Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:00.111040033Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:00.111046363Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:00.111052813Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:00.111059023Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:00.111179996Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"bff9be97-4b79-4e6b-9a6d-6f9b373920ae","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized)]","latency_us":188275,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:00.111395671Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"11.272752357s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bff9be97-4b79-4e6b-9a6d-6f9b373920ae\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:35:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bff9be97-4b79-4e6b-9a6d-6f9b373920ae\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:00.255614816Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.06153382s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.38492639Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.386088577Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.386094757Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.408101008Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.408134419Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.428590655Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.428639736Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.446754909Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.4467975Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.465080696Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.465126247Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.483409574Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.483462575Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.501336182Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.501366103Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.519581498Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.519627679Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.538263293Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.538318175Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.538670543Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.538830036Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:11.57018021Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:11.570237231Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:11.570247532Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:11.570255952Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:11.570265462Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:11.570387525Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5bbfdcb7-4813-491e-9734-c70b5c6f8f4b","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\", TimeNotSynchronized), (\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\", TimeNotSynchronized), (\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\", TimeNotSynchronized), (\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\", TimeNotSynchronized), (\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\", TimeNotSynchronized)]","latency_us":185283,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:11.57060921Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"8.68419513s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5bbfdcb7-4813-491e-9734-c70b5c6f8f4b\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:36:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5bbfdcb7-4813-491e-9734-c70b5c6f8f4b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:17.352299251Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.24803587s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.654567208Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.655030902Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.655044656Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.658452887Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.675306645Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.67535233Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.695587314Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.695621776Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.713328099Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.713372989Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.731133093Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.731170444Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.750049905Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.750097915Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.768778701Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.768830977Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.787161321Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.787200144Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.805393639Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.805442232Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.805635304Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.805762602Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.888795424Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.888863104Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.888869788Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.888885519Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.888893461Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.979726263Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.983540181Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.987272151Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.991296404Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:20.996046351Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:21.000789443Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:21.005065017Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:21.010586836Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:21.016056684Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:21.02091505Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.437002042Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.437136642Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.44606041Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.446181791Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.555469246Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.55566995Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.580032349Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.580143727Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 71.962341ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:24.593148244Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 66.684141ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:24.600529582Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.079925632Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 117.434536ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:24.090011986Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 103.148049ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:24.13034649Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.138386214Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:24.139348504Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 35.630905ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:24.232308972Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 204.637808ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:24.356849675Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 274.840462ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:24.464956219Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 80.181772ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:24.772335934Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 25.380681ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:24.830136873Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 482.458843ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:24.967125792Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 587.325013ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.034152574Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 232.635704ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.126695855Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 69.221893ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.196201425Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 26.102908ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.387000825Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 108.328565ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.504266088Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 140.80195ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.530362752Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 874.585068ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.581975352Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 522.671144ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.631972946Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 544.74631ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.672649086Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 265.864321ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.688545058Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 1.173675014s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.703723353Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 308.710887ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:25.977646588Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 644.894025ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:26.183029784Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 707.160874ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:26.249157169Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 832.327507ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:26.356420158Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 714.594199ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:26.47427705Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 1.387193327s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:26.841102113Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 1.362141512s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:26.891793009Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 1.126029441s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:26.96789798Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:26.97179155Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:26.971931878Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"5929efd7-f19c-4e78-86d0-8d8c1128aae6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56599","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":73,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 1.279976249s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:27.199293371Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 1.001673073s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:27.207864914Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 673.412121ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:28.105494857Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 1.399247817s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:28.222687449Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 605.944645ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:28.240849546Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 778.644928ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:28.267175465Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 1.202873362s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:28.49344432Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 867.799033ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:28.794107601Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 698.205155ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:28.860843699Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 758.349616ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:29.05892812Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 1.31833321s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:29.57067911Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 917.469786ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:29.633932129Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 690.791205ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:29.673970264Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 1.144695211s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:29.710103065Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 988.484092ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:29.830097641Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 697.095641ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:30.377019078Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 944.233512ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:30.564714564Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 771.453236ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:30.83280869Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 1.252228185s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:30.869682649Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 563.028261ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:30.902724689Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31\") failed: Property not found. retry in 1.335953699s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:31.088064574Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 570.735166ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:31.47952874Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 1.428867188s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:31.524073849Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 1.390541521s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:31.617989221Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 1.149992445s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:32.064199434Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 1.189179162s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:32.139316085Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 1.289347164s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:32.969481266Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 800.776493ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:33.026436074Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 833.707591ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:33.22986465Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 1.478363606s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:33.344146736Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 588.077112ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:33.842499259Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 1.034514301s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:34.080688886Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 975.577536ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:34.283378442Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:34.427581652Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"7.901917005s"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 530.473573ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:34.444358431Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 838.41372ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:34.840069844Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 1.383300034s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:34.986703841Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 817.624615ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:35.12984659Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 1.386872552s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:35.275070477Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 917.214015ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:35.692122161Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 1.249536306s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:35.960420273Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 1.006614074s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:36.386720515Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 509.637812ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:36.622846849Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 503.823257ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:36.676487708Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 1.110181188s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:37.143847985Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 1.4708605s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:37.191973339Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 1.099144692s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:37.224916659Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 1.36374802s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:37.406393127Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 799.082873ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:38.268938552Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 517.353691ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:38.339539431Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 1.239784886s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:38.679313521Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 837.169022ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:38.785776091Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 911.245781ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:38.867621332Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 1.053761619s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:39.080164663Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 857.270553ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:39.636127686Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 1.227836896s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:39.791541333Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 1.312487812s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:39.93395535Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 988.996333ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:40.148181061Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 1.27322521s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:40.506216163Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 762.983133ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:41.033786506Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 569.029818ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:41.150577584Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3\") failed: Property not found. retry in 1.429969325s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:41.262880582Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044\") failed: Property not found. retry in 1.328245481s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:41.73146599Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3\") failed: Property not found. retry in 516.054495ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:41.794509917Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200\") failed: Property not found. retry in 1.352351862s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:41.808225455Z","hostname":"buskin","pid":1590,"zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.187767963Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_0b70f5ce-d150-4dfd-8ff7-d8d885015044"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.187856699Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_2a9c3d76-6188-43a0-90ae-5dda3bf807c3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.187869225Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.187876791Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_6c7f378a-2e14-4e5e-9196-0d5d99bf9f31"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.187886044Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.187915351Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.188399777Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.188695743Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"9890458e-93d0-4bf9-9569-e68e7a435ac2","remote_addr":"[fd00:1122:3344:101::1]:50259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":23096472,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.220788352Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.249031258Z","hostname":"buskin","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"bd8c02f3-50c2-4ab9-975a-943cae0ca7b6","remote_addr":"[fd00:1122:3344:101::1]:38476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_08af5ddc-fe0d-4ade-9084-07879e7896c3 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28023,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:36:43.249254322Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934","retry_after":"69.961349ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bd8c02f3-50c2-4ab9-975a-943cae0ca7b6\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:36:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bd8c02f3-50c2-4ab9-975a-943cae0ca7b6\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:36:43.345181346Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:04.351293401Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:12.910764769Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:12.911620002Z","hostname":"buskin","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"d7197ee2-0ce3-43b2-967e-3c0558c72364","remote_addr":"[fd00:1122:3344:101::1]:38476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":29443544,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:12.914723112Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46079"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:12.92130543Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"19.872911511s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:12.956779883Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:12.957097478Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:44149"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:12.958356642Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:12.958372908Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:12.985227316Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:12.985354469Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.00636712Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.006484715Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.027213143Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.027255988Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.04745594Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.047502403Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.067587433Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.067632584Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.088141911Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.088187372Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.108689853Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.108739326Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.129279094Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.129326167Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.206709675Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.20682347Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.294973213Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.295025039Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.379741417Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.379809584Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.462017441Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.462071432Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.549899959Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.549956605Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.637044988Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.63711488Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.731605214Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.731665618Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.82135098Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.821424491Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.898983795Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.899038443Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.965998732Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:13.966062501Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.035447926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.035499011Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.110533609Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.110698509Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.111308736Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.111625956Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223314954Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223359636Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223364896Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223368852Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223372818Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223377065Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223381115Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223385222Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223395065Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223547679Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223554907Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223563804Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223569569Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.22357363Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223577417Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223581589Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223585536Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223598398Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.223604192Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.22374439Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"957a7204-c129-43f8-a81a-0adb43b212ac","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized)]","latency_us":1260189,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.224023855Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"115.885593ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"957a7204-c129-43f8-a81a-0adb43b212ac\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"957a7204-c129-43f8-a81a-0adb43b212ac\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.341737635Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.342165139Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.342172199Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.363520047Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.363564157Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.384663322Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.384703663Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.404971848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.405009461Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.425959019Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.426012506Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.447413688Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.44745582Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.466797307Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.466832211Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.487366227Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.487401909Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.507260451Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.50729499Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.526896629Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.526939989Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.545487522Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.54553526Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.565205338Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.565242567Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.585158764Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.585195477Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.604147233Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.604187508Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.623322595Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.623360218Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.643387293Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.643422337Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.663322088Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.663357705Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.683873918Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.683911869Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.703228944Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.703272192Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.722149776Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.722189489Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.742092356Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.742139813Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.742322865Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.742458562Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836203888Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836261377Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836268352Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836274268Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836278412Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836284046Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836289962Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836310841Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836316053Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836321528Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836327266Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836333903Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836339341Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836344489Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836349504Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836354595Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836360914Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836366107Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836371245Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.836471607Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"4f948a16-18a7-4a07-8a26-2fb5387ac603","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized)]","latency_us":492113,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:14.836737101Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"147.109956ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4f948a16-18a7-4a07-8a26-2fb5387ac603\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4f948a16-18a7-4a07-8a26-2fb5387ac603\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.984915874Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.985292634Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:14.985300678Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.007253672Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.007297716Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.028698168Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.028746133Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.048495872Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.048617266Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.067765959Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.067818432Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.087008404Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.087052983Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.10747269Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.107517597Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.130074958Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.130118684Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.150990517Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.151029293Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.171377287Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.171423376Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.19190751Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.191947149Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.21242488Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.212466478Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.2333045Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.233343126Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.253425353Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.253469219Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.272893273Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.272931617Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.29220571Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.292243811Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.314216962Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.314259816Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.33384657Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.333886096Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.352886208Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.35293796Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.37180373Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.371854459Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.394058007Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.394112215Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.394423176Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.394573769Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488620741Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.48868978Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488700018Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488707386Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488713058Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488720025Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488725687Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488808937Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488815472Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488820666Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488827378Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488832347Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488838526Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488843476Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488848529Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488851867Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488855307Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488864476Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.488868451Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.48894424Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c2e18cb0-3f73-4fae-b903-b0d98643f1ee","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized)]","latency_us":501354,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:15.489167249Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"181.18543ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c2e18cb0-3f73-4fae-b903-b0d98643f1ee\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c2e18cb0-3f73-4fae-b903-b0d98643f1ee\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.672274869Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.672600358Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.672608983Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.695943969Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.695990845Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.716130556Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.716177216Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.733191853Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.733236525Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.754665813Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.754731158Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.773849111Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.773892265Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.794060264Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.794112006Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.812609475Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.812644988Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.833253966Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.83329323Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.853479711Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.853523765Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.87627798Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.876317956Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.898700041Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.898748398Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.921807697Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.921858454Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.943179401Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.943239917Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.964858861Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.964912008Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.988133019Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:15.988181948Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.01178044Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.011827747Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.034977914Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.035028174Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.059774249Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.0598257Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.082405703Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.082446476Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.104369717Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.104433121Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.104657808Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.104880565Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209507594Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209576408Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.20958492Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209590667Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209596208Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209603408Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209627006Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209632837Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209638265Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209645381Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209651025Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209656238Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.20966146Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209668041Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209673432Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.20967856Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209685272Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.209690166Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.20969657Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.209818954Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"9bc205dd-1f0b-414e-85d6-51509dca44ee","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized)]","latency_us":534711,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.210015692Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"227.789942ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9bc205dd-1f0b-414e-85d6-51509dca44ee\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9bc205dd-1f0b-414e-85d6-51509dca44ee\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.440393467Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.440771527Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.440781718Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.463567537Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.463680927Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.484555771Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.484610992Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.50457565Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.504624213Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.524422783Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.52446963Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.543046979Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.54309033Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.561464077Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.561510335Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.581059064Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.58110128Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.603128259Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.603170729Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.623702782Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.623745279Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.644010307Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.644058486Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.663771941Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.663818021Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.682918029Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.682962543Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.702336215Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.702387112Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.722343179Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.72238502Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.742172631Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.742214201Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.762850601Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.762893521Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.784758085Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.784807559Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.80458019Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.804627797Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.825205359Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.825251383Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.8448763Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.844930029Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.845156684Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.845287891Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949459493Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949514741Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949521623Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949525448Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949530566Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949546973Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949552552Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949557539Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949564252Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949569605Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949574742Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.94958003Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949586695Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949592134Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949597017Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949603346Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949608306Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949613321Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.949616762Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:16.949711292Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"20c45aeb-c96e-49cd-a26a-6c882fba7771","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized)]","latency_us":506619,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:16.9499232Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"140.749005ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"20c45aeb-c96e-49cd-a26a-6c882fba7771\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"20c45aeb-c96e-49cd-a26a-6c882fba7771\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.092909497Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.09322488Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.093233027Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.116405921Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.116458759Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.138634117Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.138686647Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.158750929Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.158801949Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.179229156Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.179277017Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.199765472Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.199813662Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.219525553Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.219573076Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.238746748Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.238794027Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.257415895Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.257475371Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.277099896Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.277142685Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.296570013Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.296615229Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.315433213Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.315478364Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.334178219Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.334222751Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.352845477Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.352896534Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.371252481Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.371300613Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.389884867Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.389928996Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.408484188Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.408533511Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.427295572Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.427337863Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.446159833Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.446201619Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.465157055Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.465199112Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.484279986Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.484340531Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.484537992Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.484722648Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585054838Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585108061Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585114764Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.5851204Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585126212Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585147296Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585220078Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585225534Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585232003Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585237216Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585242541Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585249272Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585254316Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585260898Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.5852646Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585270807Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585275992Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585281345Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585286707Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.585398273Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"6098ede8-adb7-45a8-938b-a66b6f008b7f","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized)]","latency_us":489903,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:17.585619255Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"134.215546ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6098ede8-adb7-45a8-938b-a66b6f008b7f\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6098ede8-adb7-45a8-938b-a66b6f008b7f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.721524415Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.722104666Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.722112973Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.745242042Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.745290775Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.765585103Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.765632925Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.785407789Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.78544815Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.803672674Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.803717188Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.823100144Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.823147358Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.842878926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.842926112Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.86225578Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.862302028Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.881435791Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.881500395Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.901841936Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.901882174Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.922241816Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.922286986Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.943020074Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.943068994Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.965862233Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.965904721Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.985247168Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:17.985288644Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.006100809Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.006149878Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.025780077Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.02582625Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.046967019Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.047021741Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.067367988Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.067427305Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.086166711Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.086223505Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.104911647Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.104964935Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.12661471Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.126682119Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.126934646Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.127071074Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225074663Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225142492Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225150189Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.22515648Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225172718Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225179674Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225184681Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225189443Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225194768Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225199747Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225212825Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225217785Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225223381Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225229963Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225234613Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225239779Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225245329Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225249454Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225254564Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.225411235Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b815c130-b31f-4b47-8f17-6a9738b839ad","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized)]","latency_us":501165,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.225644491Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"254.743379ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b815c130-b31f-4b47-8f17-6a9738b839ad\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b815c130-b31f-4b47-8f17-6a9738b839ad\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.482162601Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.482475884Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.482484565Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.502260695Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.502315811Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.52274396Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.522880084Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.543664902Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.543726009Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.564867524Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.564939395Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.584836273Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.58489273Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.603290011Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.603336484Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.620758606Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.620814473Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.639695773Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.639755296Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.660471997Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.660596869Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.679724084Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.679774775Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.701535316Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.701598327Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.721708798Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.721758477Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.741318695Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.741368534Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.761116791Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.761179043Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.782988064Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.783032391Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.802881765Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.802942404Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.822609957Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.822669958Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.842876949Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.842930304Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.863887538Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.863939412Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.883557896Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.883686155Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.883900745Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.884033217Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981204846Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981299592Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.98130772Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981314132Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.98133473Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.98134073Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981347546Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981361683Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981367693Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981378137Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981385618Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981391515Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981397018Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981402363Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981409029Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981414673Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981422004Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981427638Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.981434941Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:18.981543057Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"937e6ad8-3128-49ef-94e1-1d3bfcfeb897","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized)]","latency_us":496747,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:18.98178234Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"223.631376ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"937e6ad8-3128-49ef-94e1-1d3bfcfeb897\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"937e6ad8-3128-49ef-94e1-1d3bfcfeb897\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.207386339Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.207701787Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.207710768Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.227984338Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.228042502Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.247320825Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.247375726Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.266583406Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.266646445Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.287519771Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.287575413Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.306495196Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.306553875Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.324605076Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.324666485Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.344053017Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.344102753Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.36289629Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.362949795Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.381869441Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.381933362Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.402071844Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.402118598Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.422068169Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.422116216Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.441976063Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.442042131Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.463820333Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.463869103Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.483862824Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.483912035Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.503472554Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.503525252Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.524035656Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.524089432Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.544493486Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.544563875Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.563938291Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.564062739Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.584078809Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.584201672Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.60396027Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.60405796Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.604375036Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.604534986Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703388663Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703464836Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703474128Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703488171Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703512322Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703519954Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703525335Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703532263Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703537729Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703543269Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703549663Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703554689Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703561448Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703567908Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703573608Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703579101Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703582964Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.70359249Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703606571Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:19.703751646Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"cb398fcb-958f-4e8a-b6fd-a0da538eb46e","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized)]","latency_us":493742,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:19.703957882Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"586.123723ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cb398fcb-958f-4e8a-b6fd-a0da538eb46e\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cb398fcb-958f-4e8a-b6fd-a0da538eb46e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.294190963Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.294497195Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.29450733Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.32073202Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.320874511Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.341191983Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.341248029Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.361169267Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.361234284Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.381571475Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.381629394Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.401486312Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.401539675Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.423255584Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.423376531Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.443936657Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.443997643Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.464771685Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.46483148Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.485009872Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.485060011Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.504885448Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.504940415Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.524527486Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.524576829Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.543958917Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.544025527Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.563713227Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.56376706Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.582733056Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.58277788Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.602641719Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.602692439Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.621646559Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.62169426Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.641815493Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.641872861Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.660811408Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.66085969Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.68092163Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.68097399Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.701438768Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.701497335Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.701774586Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.701917736Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797609183Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797685655Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797718516Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797726044Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797730272Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797737781Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.79774187Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797747673Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797754498Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797758951Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797766348Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797773099Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797786402Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797799846Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797812858Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797823547Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797830606Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797837834Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.797846506Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:20.797920111Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1161dd06-6d9f-420c-80c5-2e7ea7b682de","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized)]","latency_us":501108,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:20.798196378Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"591.846339ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1161dd06-6d9f-420c-80c5-2e7ea7b682de\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1161dd06-6d9f-420c-80c5-2e7ea7b682de\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.393026172Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.393401849Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.393419464Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.415170816Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.415225923Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.434581038Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.434633492Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.45312562Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.453171849Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.472939766Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.472995005Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.49211625Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.492168619Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.510603693Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.510655135Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.530459664Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.530512147Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.550621279Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.550744861Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.570585105Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.570634606Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.589452538Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.589501242Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.609340954Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.609387399Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.630142055Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.630203163Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.649817123Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.649869549Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.670216233Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.670269062Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.690571205Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.690621532Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.710716145Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.710765441Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.732580831Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.732630566Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.752502879Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.752556908Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.773457132Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.773511264Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.793539115Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.793605445Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.79378842Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.793912848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889538204Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889689558Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.88971399Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889736893Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.88974399Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889759732Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889767878Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.88977985Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889789986Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889794157Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889798245Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889808136Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889813911Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889821045Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889826867Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889832418Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.88983659Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.889851243Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.88985683Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:21.889873519Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"fdf147dc-a088-4572-9820-581310c59a75","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized)]","latency_us":495329,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:21.890185788Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"498.337829ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fdf147dc-a088-4572-9820-581310c59a75\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fdf147dc-a088-4572-9820-581310c59a75\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.39108049Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.391410825Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.391432013Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.412816849Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.412870437Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.432565866Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.432619529Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.452795189Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.45284949Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.472373945Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.472432408Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.491873708Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.491925357Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.511004794Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.511073834Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.530957187Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.531024126Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.552340764Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.552460717Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.572818585Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.572872249Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.592182971Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.592238323Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.612653849Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.612706491Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.632546258Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.632661478Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.65112767Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.651175887Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.671869311Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.671926434Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.691018525Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.691079473Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.712021687Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.712072595Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.732941852Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.732989337Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.752635132Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.752690464Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.772126224Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.772173138Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.791807956Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.791863074Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.792123206Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.792257835Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886254951Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886417238Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886424204Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886430204Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886435989Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886441501Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886453511Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886465821Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886473424Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886486634Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886492568Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886505591Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886511122Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886518041Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886523722Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886530556Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886545838Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886551539Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886555457Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:22.886630844Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c1f4347d-9b95-4e9f-a9ae-13a6cfb799f1","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized)]","latency_us":494074,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:22.886843811Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"878.654743ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c1f4347d-9b95-4e9f-a9ae-13a6cfb799f1\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c1f4347d-9b95-4e9f-a9ae-13a6cfb799f1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.768904042Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.769255659Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.769272722Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.793593916Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.793659973Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.815007455Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.815073307Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.83579457Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.83584996Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.854735863Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.854800327Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.875975375Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.876098686Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.898968848Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.899018153Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.918804525Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.918873583Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.938567176Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.938617213Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.959088845Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.959139978Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.980103577Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:23.980160372Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.00084538Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.000897431Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.023686802Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.02373712Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.044900272Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.044955229Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.066954858Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.067010753Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.086869409Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.086916585Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.10639871Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.108741724Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.12510162Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.125162203Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.14502496Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.145082964Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.164178288Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.164239189Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.182599585Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.182661078Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.182917791Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.183050863Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276085361Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276244294Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276252694Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.27626626Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276273563Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276282694Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276294067Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.27629851Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.27630736Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.27631172Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.27631712Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276322455Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276327611Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276332646Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276343952Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276348302Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276364719Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276370062Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.27637414Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:24.276383515Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a8146926-6b39-4d88-9f55-f973b28528ee","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized)]","latency_us":505943,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:24.276587051Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.023113406s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a8146926-6b39-4d88-9f55-f973b28528ee\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a8146926-6b39-4d88-9f55-f973b28528ee\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.304269692Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.304586331Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.304606648Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.326840399Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.326892919Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.34674144Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.346809926Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.365798971Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.36585015Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.383064827Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.383111543Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.399352Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.399406413Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.418530659Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.418653306Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.437003433Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.437060932Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.455804123Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.455861415Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.472747515Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.472796866Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.491808446Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.491857094Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.510013999Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.510070963Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.530175581Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.53022826Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.55186219Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.551913847Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.571915666Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.571976943Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.594342968Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.594402922Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.616721404Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.616845059Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.637551715Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.637608004Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.659165465Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.659219728Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.678641591Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.678687821Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.698866747Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.698924705Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.699319339Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.699439304Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.797947705Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.79801801Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798025698Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798031623Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798038054Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798043576Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798054563Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798061867Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798068964Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798074646Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798081817Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798093996Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798101121Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798108171Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798113778Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798128796Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798133296Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798137451Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798141482Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:25.798207033Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"253d7c53-20e8-42d5-b502-e2393c7034f9","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized)]","latency_us":492487,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:25.798484418Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"588.858519ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"253d7c53-20e8-42d5-b502-e2393c7034f9\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"253d7c53-20e8-42d5-b502-e2393c7034f9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.389409585Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.39037935Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.39039631Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.413554286Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.413609187Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.433390797Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.433453198Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.454717854Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.454778915Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.474804857Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.474852286Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.495576653Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.49562741Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.516285707Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.516343795Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.536915368Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.536964925Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.557691527Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.55774445Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.57757862Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.577632274Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.599704666Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.599818641Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.620323606Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.620374935Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.64187102Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.641926888Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.661034199Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.661090112Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.682306307Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.682354619Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.702305483Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.702361537Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.722547331Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.722609001Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.74378844Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.743917927Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.764715957Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.764775845Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.785415446Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.785467919Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.805421253Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.80549003Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.805652409Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.805776218Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.901902098Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.901969787Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.901991435Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.901997444Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902004448Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902011779Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902017404Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902027557Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902033014Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902046205Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.90205032Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902055815Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902060849Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902066099Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902071499Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902091056Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902096569Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902102128Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902105878Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:26.902124244Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"79ab2ea4-36cb-4803-9cde-b17485e02ffa","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized)]","latency_us":511194,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:26.902410012Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.884007344s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"79ab2ea4-36cb-4803-9cde-b17485e02ffa\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"79ab2ea4-36cb-4803-9cde-b17485e02ffa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.792581454Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.793137292Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.793161414Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.816614698Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.816680681Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.836182321Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.836248847Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.855256805Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.855307656Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.872925675Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.872975514Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.893066413Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.893136015Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.911217863Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.911264026Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.93022217Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.93028579Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.949994877Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.95004465Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.968492203Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.968535854Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.987441481Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:28.987490785Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.005625415Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.005683673Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.024717065Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.024766463Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.042587239Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.042635569Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.062331211Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.06237895Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.080592721Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.080635997Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.098818914Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.098873385Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.116191997Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.116235715Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.135668502Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.13571325Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.154596286Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.154654515Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.173065483Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.173131344Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.173312295Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.173440594Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264626098Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264685012Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264694621Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264704381Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264712556Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264719363Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264792151Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264798957Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264805688Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264813752Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264820417Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264828049Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264836167Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264842449Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264858678Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264867106Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264875581Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264881843Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.264887196Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:29.264974368Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"881f903c-c29e-4439-b02b-687a529f8223","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized)]","latency_us":470906,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:29.265193335Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.894115888s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"881f903c-c29e-4439-b02b-687a529f8223\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"881f903c-c29e-4439-b02b-687a529f8223\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.166062629Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.166693843Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.166703097Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.188735344Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.191337612Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.209440998Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.209505696Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.229863141Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.232242068Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.250135916Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.250252258Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.270466121Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.270526647Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.28890357Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.290927271Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.307663577Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.307716518Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.326519242Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.326584109Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.345387811Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.345516937Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.364250356Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.364309543Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.382854179Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.382919346Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.400962456Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.40101836Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.419768436Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.419814187Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.437938683Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.437999763Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.455982139Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.45603104Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.474096799Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.474156472Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.492458921Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.492515885Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.511181964Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.511240813Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.52956714Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.529617194Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.548277617Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.548350181Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.548538548Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.548664204Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.643836838Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.64391633Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.643924739Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.643933693Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.643941803Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.64394994Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.643958134Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.643967172Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.643973623Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.643981619Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.643989504Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.643996104Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.644004935Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.64402407Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.644030323Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.644034955Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.644041808Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.644048173Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.644054512Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:31.644120832Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"127f812c-69e6-434d-9b0e-4d601992838d","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized)]","latency_us":476374,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:31.64442406Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.002406831s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"127f812c-69e6-434d-9b0e-4d601992838d\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"127f812c-69e6-434d-9b0e-4d601992838d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.651418709Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.651732142Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.651749223Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.673466233Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.673521547Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.694547135Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.694598201Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.715109916Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.715238768Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.733385573Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.733429232Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.752187954Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.752243155Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.770876621Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.773166625Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.789583202Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.78965751Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.80720588Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.807257004Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.825669198Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.82572694Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.843688898Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.843749134Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.862160677Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.862218475Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.881419991Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.881468892Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.900476178Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.900522266Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.919532846Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.919574509Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.937832704Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.937902728Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.956326103Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.956387951Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.974462397Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.974510679Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.992105136Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:32.992164255Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:33.010759477Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:33.010812054Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:33.029379534Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:33.029456168Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:33.029650638Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:33.029812267Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118842754Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118908465Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118927206Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118933272Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118940275Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118947232Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118958416Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118965822Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118978536Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118984423Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118990086Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.118997426Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.119004336Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.11901583Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.119023274Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.119030127Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.119037459Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.119048427Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.119055788Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:33.119173361Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"913d85e2-2331-43be-9dbd-cb9540659afb","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized)]","latency_us":466378,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:33.119426766Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.876466728s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"913d85e2-2331-43be-9dbd-cb9540659afb\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"913d85e2-2331-43be-9dbd-cb9540659afb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.005347539Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.005729016Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.005740417Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.028778365Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.028850356Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.0494368Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.049494205Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.069238729Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.069284566Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.087535048Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.087590933Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.105913508Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.105976237Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.123799314Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.123854224Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.142507115Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.142633812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.161734482Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.161780298Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.179774774Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.17983022Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.198345966Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.198398401Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.217371239Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.217422456Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.23586364Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.235922273Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.2543441Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.254407457Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.272142825Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.272191116Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.289213478Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.289286491Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.308129307Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.308184986Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.326691046Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.326755416Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.343593949Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.343648643Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.361815513Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.361877577Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.378472837Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.378550342Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.3787651Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.378935486Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469536743Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469612916Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469629164Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469637967Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.46965114Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469658021Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469664593Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469671559Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469678299Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469691397Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469700088Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469716213Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469738461Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469747686Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469754023Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469762339Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469768536Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469776683Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.469783377Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:36.469893404Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"be12184b-f799-44d3-b580-1db3cfba01d0","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized)]","latency_us":463144,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:36.470152489Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.79648763s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be12184b-f799-44d3-b580-1db3cfba01d0\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"be12184b-f799-44d3-b580-1db3cfba01d0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.275203178Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.276479476Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.276488851Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.29832709Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.29840796Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.318673131Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.318731566Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.337630335Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.337676874Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.356690364Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.356748555Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.37624616Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.376302102Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.395112887Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.39516745Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.413237459Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.413283285Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.43131933Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.431381609Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.450425127Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.450476615Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.469310026Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.469355459Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.488328608Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.488389527Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.507014585Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.50706614Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.526129685Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.526177114Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.545064419Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.545115842Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.564393421Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.564462432Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.583696699Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.583746575Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.602419812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.602475294Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.623197363Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.623245018Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.642809939Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.642859187Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.663364778Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.663428305Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.663622793Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.663754Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761296528Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761374164Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761384214Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761394743Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761403152Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761483239Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761491564Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761507707Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761515695Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761527133Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761535149Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761542461Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761559056Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761565741Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761570662Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761575397Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.7615818Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761588391Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.761594503Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:39.761655461Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"75c53308-11af-40ef-ae91-dfda2dd3ec51","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized)]","latency_us":484950,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:39.76186991Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.126746956s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"75c53308-11af-40ef-ae91-dfda2dd3ec51\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"75c53308-11af-40ef-ae91-dfda2dd3ec51\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.895343778Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.895824418Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.895842071Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.917972186Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.918051951Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.938259707Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.938304877Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.957408036Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.957527527Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.977971864Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.97802864Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.996043845Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:41.996100874Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.014781519Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.014832426Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.034126806Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.034173916Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.053158341Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.053205413Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.071817345Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.071887874Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.091586471Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.091651414Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.110547284Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.110606253Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.13025087Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.130305068Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.149529578Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.149587986Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.168199645Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.168245592Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.187185113Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.187248527Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.206078969Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.206188546Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.224612287Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.224656379Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.243847967Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.243899361Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.262791698Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.262852243Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.281570696Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.281642483Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.281907023Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.282089699Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.363104728Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63540"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.364173292Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"98ee3ad8-ca5a-4b9f-8f84-2c28c8517298","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63540","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":751,"response_code":"200"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375403312Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375474619Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.37549201Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375503664Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375511557Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375520295Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375531133Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375540368Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375632976Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375644629Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375653526Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375672482Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375678615Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375686077Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375691365Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.37569699Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375705015Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.37571365Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.375722566Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:42.375777176Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b3fb4d9f-f836-4b60-a2e6-a83a681e0c01","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized)]","latency_us":478951,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:42.376008866Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.131971837s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b3fb4d9f-f836-4b60-a2e6-a83a681e0c01\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b3fb4d9f-f836-4b60-a2e6-a83a681e0c01\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.521278116Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.52177864Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.521794577Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.54374773Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.543825525Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.562401635Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.56244672Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.580134958Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.580180943Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.597840445Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.597905818Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.616783006Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.616842332Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.634844894Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.634894255Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.65377524Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.653835232Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.671739009Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.671793319Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.690087416Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.690159333Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.708462869Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.708513363Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.726514004Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.726572298Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.745164954Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.745215318Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.762888564Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.762938037Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.780851632Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.780897993Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.799310383Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.799360372Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.817488269Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.817537593Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.836061937Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.836114729Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.853922448Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.855774199Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.872708847Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.872766588Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.891014222Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.891072517Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.891267269Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.891398025Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981803082Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981889587Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981897762Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981911394Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981918125Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.98192451Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981931288Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981946138Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981954783Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981962911Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981971658Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.98197778Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.981985973Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.98199383Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.982000412Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.982005409Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.982012102Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.98201678Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.982021684Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:46.982100867Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e8f4ae38-7848-40b6-8f24-8eb2310618c9","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized)]","latency_us":459398,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:46.982364384Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.349770608s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e8f4ae38-7848-40b6-8f24-8eb2310618c9\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e8f4ae38-7848-40b6-8f24-8eb2310618c9\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:47.959693107Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"31.104994151s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.339758388Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.340069739Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.340083258Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.362343931Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.362404728Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.382987964Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.383047047Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.401029928Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.401092424Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.420139622Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.420200664Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.438964002Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.439028129Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.457132237Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.457186932Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.475083999Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.475139876Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.493856835Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.49391813Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.513160831Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.513230817Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.530986505Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.531042298Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.549443975Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.549495595Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.566921354Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.566981871Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.586406396Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.586461354Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.606199469Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.606251548Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.624990507Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.625058093Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.643363661Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.643411465Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.662734816Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.662788526Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.681328478Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.681388077Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.699434638Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.699489905Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.718924717Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.718978381Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.719179554Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.719319667Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808741722Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808801714Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808828518Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808840059Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808862765Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808938589Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808948424Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808963649Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808973512Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808982783Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.808999725Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.809011087Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.809016666Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.809023687Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.809035145Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.809042195Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.80904872Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.80905561Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.809063279Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:49.809118209Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b11910c3-4ae6-41e2-8b40-11de17c3173c","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\", TimeNotSynchronized), (\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\", TimeNotSynchronized), (\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\", TimeNotSynchronized), (\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\", TimeNotSynchronized), (\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\", TimeNotSynchronized), (\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\", TimeNotSynchronized), (\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\", TimeNotSynchronized), (\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\", TimeNotSynchronized), (\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\", TimeNotSynchronized), (\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\", TimeNotSynchronized), (\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\", TimeNotSynchronized), (\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\", TimeNotSynchronized), (\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\", TimeNotSynchronized), (\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\", TimeNotSynchronized), (\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\", TimeNotSynchronized), (\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\", TimeNotSynchronized), (\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\", TimeNotSynchronized), (\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\", TimeNotSynchronized), (\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\", TimeNotSynchronized)]","latency_us":467962,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:49.809292513Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.642454534s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b11910c3-4ae6-41e2-8b40-11de17c3173c\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:37:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b11910c3-4ae6-41e2-8b40-11de17c3173c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 903782fb-f735-42e4-99f5-7de3123abd5a, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9fce540a-e957-4b21-818e-8e7691b4bb23, kind: Service { id: 903782fb-f735-42e4-99f5-7de3123abd5a }, name: Name(\"ntp-903782fb-f735-42e4-99f5-7de3123abd5a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 41])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 24405740-5280-47ac-9ba6-32bbafb7d12f, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2d811985-d449-41d6-8da1-fa19e6f59124, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9c5f2466-2802-4fd2-9e20-5f381e90384b, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 06353358-9621-4caa-8857-0031c5e69d1b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2558bf81-619f-439f-b997-9200840f75ba, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 548940b4-5214-41c2-abeb-e62ada780a97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cc0858fd-163c-4f3b-8fc3-445d0df78e3c, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1799fea1-4b4d-4e07-a51c-330407b4d185, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b76e30e-a01e-4208-8ad2-11a799c19622, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External } } } }, OmicronZoneConfig { id: 97b27fdb-af4b-43da-ab01-527baa56f359, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5070c443-fb71-4cf2-aba8-d9052fd52960, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.462746819Z","hostname":"buskin","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.463084391Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.463097619Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2cf9c6bb-9d77-4ff6-9eff-4f633dd8ba50, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.484714694Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.484771329Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a0ff8ea7-90b6-41b5-8781-fe3c2645f899, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.503922938Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.504051343Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcb9494f-42ec-4d91-a912-48c9b354aa2f, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.521752315Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.521817884Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f94d94b3-b5c8-4f46-bd6e-d85ba6c0a200, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.537493812Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.537554647Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b70f5ce-d150-4dfd-8ff7-d8d885015044, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.55660888Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.556661306Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2a9c3d76-6188-43a0-90ae-5dda3bf807c3, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.57291006Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.572958811Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c7f378a-2e14-4e5e-9196-0d5d99bf9f31, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.592391345Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.592448919Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08af5ddc-fe0d-4ade-9084-07879e7896c3, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.610171971Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.610228945Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 24405740-5280-47ac-9ba6-32bbafb7d12f, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.6286004Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.630942553Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 764ef011-aa4c-433d-93c2-908f5dcbfffc, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.647303273Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.647359618Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 06353358-9621-4caa-8857-0031c5e69d1b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.664602919Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.664679806Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0cea6f7b-fedb-469c-abf8-beab25e1cf0e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9b351946-585e-4029-8f1b-9f69550dce71 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.681109363Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.681170302Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1799fea1-4b4d-4e07-a51c-330407b4d185, dataset_name: DatasetName { pool_name: ZpoolName { id: 478a6e43-eea9-40ca-b6f4-6a492473e177 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.698104416Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.698166443Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 19dd8dec-02b3-4cf8-9bf1-f4997594f8a4, dataset_name: DatasetName { pool_name: ZpoolName { id: e3118907-da30-4e85-a767-c5a42a2e46ca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.715405618Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.715465975Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b76e30e-a01e-4208-8ad2-11a799c19622, dataset_name: DatasetName { pool_name: ZpoolName { id: d280116a-a989-463f-b7e1-373c7fcb5463 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.732759639Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.732815374Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8db2c6b0-a4df-4d14-aaf2-260eb2e11da0, dataset_name: DatasetName { pool_name: ZpoolName { id: 661f9fd6-5e25-45d3-955c-06a9af9dad35 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.75116556Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.751218689Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 97b27fdb-af4b-43da-ab01-527baa56f359, dataset_name: DatasetName { pool_name: ZpoolName { id: 9962603d-6bc6-41a3-a346-9382c9e6bfb4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.767270289Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.769239317Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d3f0da5-7774-4e18-b4f8-104a522febfd, dataset_name: DatasetName { pool_name: ZpoolName { id: ff581678-2bea-470b-9448-890977b6d596 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.786218134Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.786271825Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5070c443-fb71-4cf2-aba8-d9052fd52960, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b1821a9-dc17-4642-9053-2192b3ae698e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.803768589Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.803891917Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c259c60-4408-410b-bb67-a50d63e5d9eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 3c2c8854-3269-4025-ae74-138b796a9368 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.820453453Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.820506939Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.820702478Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.82083419Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.907254189Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.922073939Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.92239005Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.922476274Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.970181992Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.970242555Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.970313414Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.970318233Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.970322235Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:53.970334836Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.133299075Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.135031296Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.193071729Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.193314585Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.193373283Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.193420553Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.19345331Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.193468648Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.204033774Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.205533088Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.210898669Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.215698764Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.220402462Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.225493728Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.230472575Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.235385795Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.239912528Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.362452761Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.36742705Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.372628154Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.37708851Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.381285156Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.385843745Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.390786313Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.395775363Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.400885304Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.405617081Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.410206809Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.415656642Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.421209252Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.427892816Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.472018795Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.472883464Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.480032022Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.487561994Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.493192744Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.499378105Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.506154445Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.561093992Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.609461649Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.665102205Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.698426582Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.700378715Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.700412748Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.754606908Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 214, 17]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.754874533Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.75933215Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.759361579Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.835060182Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 189, 97]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.835153119Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.838009948Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.838028144Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.888039238Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 229, 178]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.888104283Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.890607778Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.890622328Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.942626066Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 183, 97]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.942705333Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.945354846Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.945369471Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.99552341Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 141, 184]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.995586975Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.996037641Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:54.996115467Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.002865897Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.00543285Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.032054021Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.032109381Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.032424118Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.032468303Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.032642128Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.032657223Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.032875787Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.032890872Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.033805048Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.036241884Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.062901107Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.066652879Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.069409806Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.069429964Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.069505106Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.071854567Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.072002584Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.07965397Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.1596726Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.164251126Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.170479853Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.176496787Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.182157892Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.195311905Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.208421256Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:55.281801434Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:57.865115684Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:57.865413319Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:57.879392802Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:57.880185765Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:57.98481588Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:57.984955505Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:57.984964092Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:57.996371016Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.007722532Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.008139982Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.008167076Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.017921263Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.018044322Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.029017678Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.049713538Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.049941439Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.060925202Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.061158664Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 39.023672ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:58.833941305Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 44.111338ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:58.846038652Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.853394428Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 27.157693ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:58.868922864Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 52.779663ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:37:58.88424142Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:37:58.884339738Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Updated Peer Addresses: {}","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:05.879412663Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:514"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 139.879177ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.070591415Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 90.409858ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.236532028Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 57.043264ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.461255391Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 63.265053ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.518954859Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 67.480483ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.568303593Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:06.572674716Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 69.188772ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.60922863Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:06.611371142Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 85.653282ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.636864775Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:06.6372571Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 47.297158ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.656795764Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 26.804753ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.670188372Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:06.670788949Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 143.402023ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.695878332Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 125.614558ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.719541214Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 100.7658ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.738229996Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 67.372946ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.752252163Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 110.845012ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.765880331Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:06.765960564Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 210.579201ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.812276521Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:06.812448325Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 279.686ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.865409757Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:06.865926191Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:06.866046268Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 69.662399ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.888460215Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 131.470352ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.903962739Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 138.833506ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.918986335Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:06.919062968Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 230.491227ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.940653129Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 339.441487ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.953581203Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 288.075829ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.967093442Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 117.576657ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.98038555Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 297.561726ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:06.992219066Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:06.992338816Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 75.433344ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.010332376Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 433.956749ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.03758481Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 126.461549ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.055808763Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 167.296923ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.074400679Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:07.074855573Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:07.074912818Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:07.074985823Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:07.081703696Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 163.079577ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.10086198Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 536.994912ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.112329416Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 28.329663ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.125371595Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 34.199942ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.133971142Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 58.110875ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.142835831Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 47.194055ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.153794719Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 523.446487ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.166036661Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 61.770992ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.176020739Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 87.337124ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.186735607Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 277.854597ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.199527084Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 511.782101ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.211705586Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 138.548045ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.222079386Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 52.329831ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.231914862Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 286.245111ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.251785757Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 488.280632ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.263401991Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 280.670227ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.278069584Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 104.542665ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.289326466Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 241.160031ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.303043463Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 61.277173ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.313255514Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 838.469778ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.324441413Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 59.643463ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.333993514Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 297.434367ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.347252485Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 48.188521ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.359734595Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 124.45042ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.375496133Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 70.053737ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.388486884Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 121.678194ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.398901136Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 95.687695ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.411776534Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 494.961985ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.427349603Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 145.142892ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.437672055Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 206.689356ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.472295815Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 1.019135992s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.487022801Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.187361612s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.497095875Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 265.101178ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.513656935Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 156.817589ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.527397423Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 462.591314ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.543126261Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 446.538976ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.558804498Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.152001987s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.576250485Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 850.684724ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.590331018Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 290.306316ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.601415214Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:07.601875619Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:07.602877047Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 449.778546ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.665281879Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 844.821637ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.682148014Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 389.835535ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.700129214Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 486.196454ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.71654968Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 545.033182ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.732244416Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 632.435825ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.745050615Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.0813651s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.767933564Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 950.179983ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.798124093Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 48.047869ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.885040676Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 539.974881ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.909792436Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 648.211147ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.938497462Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 95.699202ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:07.950097567Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.092666232s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.019927632Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 515.222582ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.034188384Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 126.215197ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.058499565Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 468.282655ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.107668517Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 558.001717ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.135819398Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 738.324624ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.182897406Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 420.576365ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.201265694Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 453.160055ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.22450943Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 694.686012ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.301052611Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 1.226553406s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.39603236Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 1.433480846s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.462261404Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 756.390291ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.477667081Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 631.247261ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.52345774Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.010600451s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.544588637Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.074171607s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.565233281Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 1.289826642s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.593567976Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 793.390279ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.607816878Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 896.2215ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.637855252Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 880.147673ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.694729782Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.264573187s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.708267383Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 1.026397567s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.721427548Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 746.205707ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.746568119Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 827.31983ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.765534119Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 700.008784ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.867318834Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.007726016s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:08.935752027Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 903.850204ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.010151771Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.250871541s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.144575854Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 921.667894ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.259273095Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 723.971246ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.278185444Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.174629044s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.417189936Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 805.656557ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.507776432Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 750.816585ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.550670457Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.34629176s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.570785399Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 713.957278ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.583832995Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.380713327s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.597799283Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 1.098089554s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.60987515Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 1.03086215s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.641256145Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 828.229151ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.656547559Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 1.450355572s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.765100344Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 641.919943ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.900157969Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 1.478519056s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.912894545Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 630.205891ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.93086504Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.457317497s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.958323732Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 680.523834ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:09.98959699Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 1.248803922s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.015735202Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 865.947958ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.200746123Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 795.562877ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.334953152Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 978.002864ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.347949999Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 671.947748ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.359577277Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.291899316s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.411881214Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 551.194593ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.50105694Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 1.261102663s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.556729395Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 855.467338ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.575242914Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 569.329722ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.608081324Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.407717962s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.683283083Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 1.135939239s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.694781855Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 514.631336ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.720558439Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 930.772057ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:10.931691236Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Updated Peer Addresses: {[fdb0:edc4:a262:183c::1]:12347}","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:11.017506874Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:514"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 549.693184ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.031389568Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.087561375s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.045998171Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.336904093s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.065061725Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 650.186111ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.081864253Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.497198889s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.144729614Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.200599932s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.194737769Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 691.448402ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.232095911Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 792.610356ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.256445052Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 1.424763873s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.281648928Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 527.501733ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.347681409Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 597.795772ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.411888379Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.068028197s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.435915171Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 516.701739ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.45130494Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.431739775s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.59533564Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 746.154241ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.722130557Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 1.281235347s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.747403486Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 1.196025704s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.834635342Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 1.452200118s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.845829363Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 889.570605ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.880156122Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 1.348634641s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.894207656Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 1.298709943s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.938983427Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 1.498243137s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:11.98309503Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 982.659081ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.025170414Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 1.226781903s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.0653654Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 961.692568ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.106222991Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 743.56419ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.147072181Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.10741653s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.417259686Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.478701358s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.436332911Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.063728474s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.480568588Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.0945944s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.516548848Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.352273138s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.656876665Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 1.322023102s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.722730935Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.081323901s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.783781276Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 803.051496ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:12.961166497Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 1.046962187s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.021061592Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 918.482532ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.042251624Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 1.053081734s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.055421334Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 763.07248ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.067841997Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 843.40873ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.080308451Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 723.338811ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.25190934Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 1.280199128s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.264762193Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 523.565626ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.305768687Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 557.035586ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.318297522Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:13.325200189Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57425"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:13.409154772Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"5532722e-98d3-40e7-b9bc-1741600b8e40","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57425","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":78556,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 896.295311ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.495068599Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.413313895s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.537992667Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.253968164s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.557529381Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 661.435711ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.624085408Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.277277204s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.776061759Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 931.897087ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.840004656Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 858.968025ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.849651696Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 597.790211ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.876803824Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 820.686629ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.886212819Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.096150381s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.926072116Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 931.294572ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.935488031Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 686.169256ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.970860927Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 929.081303ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:13.985399168Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.476638752s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.019093515Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 722.820682ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.054739097Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 1.343228915s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.078383356Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 724.350595ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.118745955Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 748.222639ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.331622724Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:14.332572551Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:14.333138055Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 1.356092351s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.402432487Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.105728818s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.485650073Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 1.163738159s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.555959444Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 35.107811ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.650421306Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 744.686285ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.666292677Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 79.279477ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.692831402Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 990.227914ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.715174711Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 924.363004ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.723350967Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 122.128013ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.780955079Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 866.708561ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.788380168Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 1.159380121s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.796098644Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 696.649053ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.820091351Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 1.464958384s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.851770092Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 574.99306ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.875263597Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 561.827915ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.911769519Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 1.331336503s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.923452179Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 601.367196ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:14.960143251Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 727.721244ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.031369163Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.470177113s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.062817029Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 763.485633ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.088017134Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 647.981511ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.42103524Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 586.685399ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.430067335Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.354950832s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.459389093Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 551.221259ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.483879251Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.478773735s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.503283153Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 594.738849ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.526292957Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 731.680575ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.569934091Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 991.057132ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.600709272Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 1.40369119s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.656662956Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 779.246953ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.664192368Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 1.383103168s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.713755847Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 645.139942ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.727922879Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 1.229088702s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.767177713Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.042671885s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.774601543Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 953.45243ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.860609982Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 835.4996ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:15.964829096Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 906.207183ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.024739371Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 1.422754813s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.04358777Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.238699646s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.078759111Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.150229194s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.145617484Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 1.406508413s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.264957772Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 712.465185ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.311118773Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 1.420064755s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.326032303Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 967.833585ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.381984717Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 503.577695ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.451927721Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.205372948s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.541617644Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.397189586s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.600931941Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 966.660061ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.809045475Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 578.118907ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.822896881Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.228115722s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.830343321Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.103667123s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.837628376Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 687.279756ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.939981068Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 668.443201ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.96508779Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 546.52612ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:16.990318485Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 1.33345306s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.004968222Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.11618951s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.032375323Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 886.231004ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.069807425Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 1.150458777s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.105515938Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.280840278s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.304128253Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.182607952s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.326027381Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 571.855161ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.357902338Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.321751488s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.40923544Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 976.110138ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.475343133Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 599.817215ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.545333547Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 1.080484019s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.635589463Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 808.476678ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.642635173Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 914.870927ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.679910292Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 1.31718727s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.754625524Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.467925282s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.762192447Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 833.830149ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.78298198Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 1.044925547s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.936788374Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.494555691s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.949158285Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 1.022848911s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:17.964577437Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 944.608517ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.007030954Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 947.227307ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.067513231Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 701.081546ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.153601732Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.000999257s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.160788396Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 1.281569942s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.263911265Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 928.242302ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.348107922Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 758.046453ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.45950177Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 881.937773ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.46559918Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 665.943687ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.516031418Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 705.941351ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.591973008Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 1.099282595s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.601502994Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 831.123063ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.624925698Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 1.059490268s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.730495563Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.381025157s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.74265032Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.199098579s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.864818783Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.161767779s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.961339171Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 1.438136411s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.990885584Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 1.33472828s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:18.998771924Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.148541498s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.023540508Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 974.183953ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.082968452Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.292421328s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.263936053Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 842.664405ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.27256411Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 1.22939419s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.2804799Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.277535318s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.288690527Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 878.256844ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.29712995Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 931.868828ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.306128034Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 1.318378805s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.356219385Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.438126251s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.453578343Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 647.259098ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.465858423Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 1.48454064s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.555188118Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 715.807285ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.710574677Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 1.325386909s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:19.800156628Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 914.136243ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.068563702Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.098710328s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.078015637Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 1.140501108s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.124514356Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 953.175229ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.134853401Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.078063812s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.144737767Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.390225533s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.154429267Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.345294068s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.183324005Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 1.003483147s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.193938347Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.495033189s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.248429768Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 738.304181ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.344584199Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 976.943393ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.438095869Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 1.145318534s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.447941743Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 1.355450335s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.521391536Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.186496048s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.566101085Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.233197886s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.57642785Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 1.474482078s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.689919085Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.284265506s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.904280478Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 915.736543ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:20.99436263Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 1.402648848s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.051602484Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 1.351512176s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.096110408Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.291966277s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.106342762Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 1.006597999s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.136252542Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.302863058s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.189455763Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 1.054707008s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.207561896Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.250074325s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.234598042Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 1.159318716s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.277026948Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 1.163207789s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.427971287Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.222040153s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.543250082Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.394516366s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.558226343Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 837.867125ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.605826768Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 911.112751ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.759298463Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.009132833s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.77273401Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.28316857s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.827517688Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 1.468375929s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.891581807Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 863.832044ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:21.926344449Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 693.899556ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.157441863Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 591.04374ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.184398146Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.301957043s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.206296305Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 716.296888ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.275725537Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.21093321s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.411219127Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 1.098366426s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.447900849Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 688.010151ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.457203451Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 1.207132086s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.469517601Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 803.57816ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.48221397Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 869.222861ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.500704422Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.335214195s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.52479351Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 1.413053965s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.606755207Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.225870701s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.690426593Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.455664483s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.784325965Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 510.285737ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.797608514Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 788.031408ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.812034183Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 640.724875ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.824598389Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 1.31197751s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.863867244Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.111107598s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:22.96863726Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 1.497088204s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.001537647Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.17072417s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.128568063Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 512.84559ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.165035874Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 688.153443ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.296318794Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.018443271s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.320181388Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 626.922216ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.372366026Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.450128029s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.385065915Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 715.001127ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.477041071Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 554.066001ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.518232779Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 849.841442ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.556889869Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 958.356999ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.609728733Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 774.457503ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.632237036Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 594.004877ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.685867797Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 1.007174194s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.695434785Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 687.277447ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.870445242Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 753.860382ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.926849196Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 506.894477ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:23.993578607Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 516.328131ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.007368531Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 926.437365ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.027776947Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 565.662085ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.082659796Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 974.122095ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.092316686Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 781.202733ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.187685718Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 590.452791ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.203749604Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 982.81052ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.253076418Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 726.751854ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.293010517Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.439110692s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.312196484Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.108029022s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.348720796Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.067387132s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.416971381Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 1.376897835s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.424966453Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 1.474444378s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.51308848Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 822.816889ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.52463082Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 697.9537ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.539309318Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.491547539s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.569435524Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 1.121708698s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.582677275Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.377872429s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.667535938Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 982.7744ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.69352399Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 1.13700989s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.714298884Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 545.938971ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.80898424Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.002961153s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.847654721Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 1.139568815s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.972603117Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 530.47506ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:24.985520531Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 912.725715ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.033444374Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.271403611s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.079698878Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.251977405s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.253377978Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 680.444866ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.263910808Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 792.061623ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.359657031Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 1.371132927s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.371589493Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.498624418s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.469132057Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 626.231057ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.499054949Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 1.484497495s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.529287978Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.22509955s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.689599263Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 729.923301ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.71886977Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 760.776588ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.765299669Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 870.454509ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.820517208Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 838.137574ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.864806888Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 682.978066ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.874912948Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 682.944139ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.958424482Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 1.289251486s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:25.970863826Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 993.087171ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.001149276Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 1.186547552s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.061149084Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 1.337063755s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.076710319Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 843.808719ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.12589345Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.315447697s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.139724695Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 856.783757ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.164233684Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 1.409618946s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.363145569Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 961.398419ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.458668967Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 860.398937ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.515141404Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 654.101185ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.534663359Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 760.059774ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.566671039Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b\") failed: Property not found. retry in 1.460620079s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.650374238Z","hostname":"buskin","pid":1590,"zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 596.65965ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.700409888Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.014278991s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.712555055Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 606.846016ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.751231177Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 708.006332ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.931671661Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2\") failed: Property not found. retry in 1.379079829s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.976710007Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd\") failed: Property not found. retry in 1.341183673s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:26.985339414Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 776.603266ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.002603178Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 572.852377ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.023255919Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960\") failed: Property not found. retry in 1.054615542s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.03166204Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b\") failed: Property not found. retry in 1.059949341s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.197727516Z","hostname":"buskin","pid":1590,"zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba\") failed: Property not found. retry in 727.543976ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.256601578Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97\") failed: Property not found. retry in 790.362463ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.267724832Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f\") failed: Property not found. retry in 1.490542943s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.310162569Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 693.388525ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.335133709Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7\") failed: Property not found. retry in 1.439171665s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.369608275Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 559.321247ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.385055577Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622\") failed: Property not found. retry in 824.37166ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.424767122Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc\") failed: Property not found. retry in 1.001606727s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.434130386Z","hostname":"buskin","pid":1590,"zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c\") failed: Property not found. retry in 1.399561526s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.461917839Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359\") failed: Property not found. retry in 876.791594ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.605755859Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e\") failed: Property not found. retry in 1.370976167s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.65100427Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185\") failed: Property not found. retry in 1.313819775s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.735864565Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0\") failed: Property not found. retry in 612.479ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.780923842Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb\") failed: Property not found. retry in 1.49940106s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.791925283Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4\") failed: Property not found. retry in 1.214624394s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:27.957625911Z","hostname":"buskin","pid":1590,"zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124\") failed: Property not found. retry in 691.402288ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:28.039054667Z","hostname":"buskin","pid":1590,"zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.33003507Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_61da6792-cf31-45c6-bbeb-ff328b88b6a7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330125972Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_cc0858fd-163c-4f3b-8fc3-445d0df78e3c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330140392Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_97b27fdb-af4b-43da-ab01-527baa56f359"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330153542Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_784ca024-14c4-4f3f-a397-cf4fa9131ca2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330163668Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_1799fea1-4b4d-4e07-a51c-330407b4d185"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330169623Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_6d3f0da5-7774-4e18-b4f8-104a522febfd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330175633Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_2558bf81-619f-439f-b997-9200840f75ba"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330181803Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_548940b4-5214-41c2-abeb-e62ada780a97"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330187733Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_0cea6f7b-fedb-469c-abf8-beab25e1cf0e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330196434Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_7c259c60-4408-410b-bb67-a50d63e5d9eb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330202624Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_24405740-5280-47ac-9ba6-32bbafb7d12f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330213844Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_2d811985-d449-41d6-8da1-fa19e6f59124"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330219894Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_8db2c6b0-a4df-4d14-aaf2-260eb2e11da0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330244225Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_06353358-9621-4caa-8857-0031c5e69d1b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330258115Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_9c5f2466-2802-4fd2-9e20-5f381e90384b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330264185Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_19dd8dec-02b3-4cf8-9bf1-f4997594f8a4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330273735Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_4b76e30e-a01e-4208-8ad2-11a799c19622"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330279936Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_764ef011-aa4c-433d-93c2-908f5dcbfffc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.330285836Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_5070c443-fb71-4cf2-aba8-d9052fd52960"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.334050591Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.336117969Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.346194718Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"891817e3-dc92-4830-a5f8-d03999bb5fd3","remote_addr":"[fd00:1122:3344:101::1]:44149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":35835887,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.350296282Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1212"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.358236973Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.358439647Z","hostname":"buskin","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.372543149Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:679"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.380337417Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:692"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:29.413848101Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:803"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:31.187703529Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:31.196451418Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:31.197504832Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:31.283665705Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:31.4441185Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:31.62365688Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:31.686052571Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:31.836632331Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:32.112083306Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:32.200104081Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:32.242972548Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:32.502155552Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:33.003626366Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:33.203694953Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:33.602438406Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:33.683419891Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:34.206800383Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:34.256825913Z","hostname":"buskin","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"65.871645745s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:34.338109954Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:34.891561012Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:35.209391532Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:35.684164728Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:35.922560308Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:36.212226207Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:37.206171528Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:37.215388469Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:37.683008471Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:38.218026788Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:38.793755504Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:39.220422893Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:38:39.682827406Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:40.097506563Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:40.223172186Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce40678a-3256-41be-b175-a8516c162f00): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T22:38:41.225668742Z","hostname":"buskin","pid":1590,"component":"ProbeManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/20971dba-ebe9-4b17-b24a-98610ff87796/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:41.749631188Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7311468f-b677-4e0c-aad0-7af87eb6a9d7\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 22:38:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7311468f-b677-4e0c-aad0-7af87eb6a9d7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.293754603Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.442465471Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.442574163Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"739c728f-146b-4f5a-b55e-ccda3530e7a3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39404","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.776715285Z","hostname":"buskin","pid":1590,"component":"NexusNotifierTask","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.904900645Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39331"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.905821046Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.989982623Z","hostname":"buskin","pid":1590,"uri":"/inventory","method":"GET","req_id":"9a37e135-4c99-4254-8b63-cd3d292fbe1a","remote_addr":"[fd00:1122:3344:101::a]:39331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":85035,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.991304173Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.991378215Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.99379576Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:43.99465636Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"460696a1-c1e7-4427-baf8-f816c528cf50","remote_addr":"[fd00:1122:3344:101::a]:39331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4059,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.103315995Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:58733"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.103675413Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c540621a-daa5-4a48-b672-7151b4fbd4ee","remote_addr":"[fd00:1122:3344:101::c]:58733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":338,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.900265419Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:42261"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.904117597Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d8afb846-3ebe-45ba-8f6f-dc49839d7d54, kind: Service { id: 764ef011-aa4c-433d-93c2-908f5dcbfffc }, name: Name(\"external-dns-764ef011-aa4c-433d-93c2-908f5dcbfffc\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 141, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f863fa93-b24e-4b4b-bf5f-1391b531b9ce, kind: Service { id: 24405740-5280-47ac-9ba6-32bbafb7d12f }, name: Name(\"external-dns-24405740-5280-47ac-9ba6-32bbafb7d12f\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 178])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06d8941a-2ca7-47f7-b5b6-45cf5f908752, kind: Service { id: 784ca024-14c4-4f3f-a397-cf4fa9131ca2 }, name: Name(\"nexus-784ca024-14c4-4f3f-a397-cf4fa9131ca2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 47557d49-fd34-48ae-b53b-adacfe92cdbd, kind: Service { id: 61da6792-cf31-45c6-bbeb-ff328b88b6a7 }, name: Name(\"nexus-61da6792-cf31-45c6-bbeb-ff328b88b6a7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f349659c-3e0f-450e-b7bb-cfac4867b61d, kind: Service { id: 2d811985-d449-41d6-8da1-fa19e6f59124 }, name: Name(\"nexus-2d811985-d449-41d6-8da1-fa19e6f59124\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 97])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.904182338Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.904227809Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.904235959Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.904248889Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.90426717Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.904292821Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.904315111Z","hostname":"buskin","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b31bfee2-8844-459e-a053-3a7ec9cbc77c","remote_addr":"[fd00:1122:3344:101::c]:42261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4010,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:45.905045598Z","hostname":"buskin","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:865"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:46.154333826Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:57317"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:46.154427328Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:46.241346459Z","hostname":"buskin","pid":1590,"uri":"/inventory","method":"GET","req_id":"75738ea1-d7e6-4bb1-a703-340dd22a42c3","remote_addr":"[fd00:1122:3344:101::c]:57317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":86935,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:46.241769118Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:46.24183467Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:46.242201648Z","hostname":"buskin","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:38:46.242464404Z","hostname":"buskin","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"21b9859c-3654-42c2-881e-4bf5bcb06f47","remote_addr":"[fd00:1122:3344:101::c]:57317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":827,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:11.8899286Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:50394"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:11.980908822Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b7ba141a-9200-4389-9444-9d147ee2ce45","remote_addr":"[fd00:1122:3344:101::a]:50394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90928,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:11.981050335Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:36492"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:11.984517885Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"bda11143-78fb-45f3-b172-66f96c9bbe02","remote_addr":"[fd00:1122:3344:101::a]:50394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3259,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:11.986042329Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:35820"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:12.004168612Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:12.004295255Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:12.005060642Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:12.005765798Z","hostname":"buskin","pid":1590,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:12.005788589Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"PUT","req_id":"8f4cc8e3-b5d7-4238-846b-c1847f3f8221","remote_addr":"[fd00:1122:3344:101::a]:50394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1984,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:12.030804249Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6bc4d56a-6b87-4142-9c07-2fdf0c155878","remote_addr":"[fd00:1122:3344:101::b]:36492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49612,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:12.034092494Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"020621f3-3e69-4370-b9a3-6de8183ddc86","remote_addr":"[fd00:1122:3344:101::b]:36492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":631,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:12.077867821Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"7cf2d221-6c63-46f7-9805-8f348fb7648f","remote_addr":"[fd00:1122:3344:101::c]:35820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91707,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:12.078326642Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a391ac2d-9d33-478b-a591-b8e307b6ddea","remote_addr":"[fd00:1122:3344:101::c]:35820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:13.473412491Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63887"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:13.473538757Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"c7308067-9585-44da-9669-ba24863a294a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63887","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:42.067940977Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56649"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:42.068059159Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:36729"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:42.118049058Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c8e4d099-6bef-46d9-80a6-728897012f7e","remote_addr":"[fd00:1122:3344:101::a]:56649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50040,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:42.124025164Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"0306ce45-7718-4415-8ddd-5e82ca0ddefd","remote_addr":"[fd00:1122:3344:101::a]:56649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":643,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:42.168993909Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"902c8cca-36c9-4683-a506-f08ce6466f10","remote_addr":"[fd00:1122:3344:101::b]:36729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":100831,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:42.169340686Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"43a6060d-5961-4ecd-a3e8-a213124bdecd","remote_addr":"[fd00:1122:3344:101::b]:36729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:42.355332353Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36319"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:42.400306728Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"8e55e7e9-5661-4b80-ae33-87b184ce1a7a","remote_addr":"[fd00:1122:3344:101::c]:36319","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44891,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:42.400552684Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"6d0580ee-2536-4f2d-ba30-acc9adc3cea7","remote_addr":"[fd00:1122:3344:101::c]:36319","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:43.503228442Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:39:43.503332654Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"4a356144-98ba-46d2-b281-d87f8324bd78","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36483","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:10.250071397Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:45107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:10.259792869Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"286f9d15-f938-4f9f-91ad-8d695dc22b5e","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9511,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:11.766533812Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:37865"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:11.819294604Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"f66403e9-c128-4aab-b711-4eb000684438","remote_addr":"[fd00:1122:3344:101::a]:37865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52724,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:11.81957107Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"dccb0220-743b-46be-b090-2ded0df25895","remote_addr":"[fd00:1122:3344:101::a]:37865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:11.864251868Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56912"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:11.910386169Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"fa64789a-0162-40cb-b393-4f0bebe1f7c5","remote_addr":"[fd00:1122:3344:101::b]:56912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46024,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:11.910665285Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"89a0314a-d3f0-4c9d-b5d8-0ae02431147b","remote_addr":"[fd00:1122:3344:101::b]:56912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:12.08822874Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51131"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:12.135062017Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a83fe39e-3423-41f0-98b2-def341387dd6","remote_addr":"[fd00:1122:3344:101::c]:51131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46799,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:12.135294712Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"0e344732-73a8-4c79-bdf0-4465a32d812d","remote_addr":"[fd00:1122:3344:101::c]:51131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:13.537687348Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:13.537797631Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"e9569d30-bee6-4c97-a337-008e241dc42b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61134","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:40.253064318Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"35cbfc31-de07-4dcd-8298-52a0af3be8fc","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":973,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:41.950324049Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:64386"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:42.000502232Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b64ff0ff-e33a-4ccb-aad4-800753739b32","remote_addr":"[fd00:1122:3344:101::a]:64386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50116,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:42.000747608Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8880e3b9-8f7f-4fff-ba38-fc609f2e2139","remote_addr":"[fd00:1122:3344:101::a]:64386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:42.080988576Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48224"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:42.127450374Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"38da15da-bd58-4d4e-8458-4e996445f2e4","remote_addr":"[fd00:1122:3344:101::b]:48224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46366,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:42.127741661Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"68778ceb-4c9d-4cba-8e57-8f12460e3f3a","remote_addr":"[fd00:1122:3344:101::b]:48224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:42.30466166Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37571"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:42.350963445Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a64db449-0eda-43fb-942c-61b95fd6c916","remote_addr":"[fd00:1122:3344:101::c]:37571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46203,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:42.351223971Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"21069e95-c350-40f2-8157-49c5b63409fe","remote_addr":"[fd00:1122:3344:101::c]:37571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:43.575565201Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33766"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:40:43.575687563Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"8056e7be-d315-4a0a-be41-b3d6f44628b1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33766","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:10.253120388Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"48c4d03f-169c-4c30-9e29-fcc32f3e699c","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":563,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:11.75896393Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:40394"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:11.811579458Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b8c08661-11d4-45ed-958e-38c8cab91a18","remote_addr":"[fd00:1122:3344:101::b]:40394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52525,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:11.811903656Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b998b576-5da4-4869-aef6-859c83475f1b","remote_addr":"[fd00:1122:3344:101::b]:40394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:11.969846474Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:51801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:12.017915499Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"bdddb36a-46b2-40e8-8f39-f74921a64014","remote_addr":"[fd00:1122:3344:101::a]:51801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47965,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:12.018024821Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:43710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:12.062657788Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"23ce2fb1-6895-4631-98ea-eddf55bceca8","remote_addr":"[fd00:1122:3344:101::c]:43710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44561,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:12.063446076Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"93bc0e25-1632-41a3-a2ba-cc7f6dc108b2","remote_addr":"[fd00:1122:3344:101::a]:51801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":672,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:12.063674061Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"434f2b46-abe6-40b3-a8e2-862044a02998","remote_addr":"[fd00:1122:3344:101::c]:43710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:13.610519787Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43550"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:13.610646459Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"22a9392a-e5d7-436f-a419-91e4bccc88ff","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43550","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":109,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:24.080202665Z","hostname":"buskin","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:40.25327076Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"5ded962e-9fee-4013-bf46-03a11447ef71","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":732,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:41.908366732Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42486"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:41.954280217Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"7f8ef941-c45b-42d4-b334-b69917ed962d","remote_addr":"[fd00:1122:3344:101::b]:42486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45849,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:41.954509333Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"77597210-ab1f-4a02-868a-446795d45f72","remote_addr":"[fd00:1122:3344:101::b]:42486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:42.13397555Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46563"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:42.182794233Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"063b1a6e-00b4-4785-a6b2-e9f3b2a2b873","remote_addr":"[fd00:1122:3344:101::a]:46563","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48725,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:42.183046608Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"57a36a74-3ffa-4b1a-b7f2-9f7233b0e12b","remote_addr":"[fd00:1122:3344:101::a]:46563","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:42.227773437Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:59474"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:42.273511429Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ae80459f-63b3-42a5-bbf0-2e31f9612b87","remote_addr":"[fd00:1122:3344:101::c]:59474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45639,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:42.273753074Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8372b86a-47fc-486e-b2b1-ad6264a009aa","remote_addr":"[fd00:1122:3344:101::c]:59474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:43.64456565Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64257"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:41:43.644660862Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"d46c8dfd-5083-4a2d-ba47-193e3e2006c8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64257","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":94,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:10.264725819Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"db513799-14cd-4dfc-a558-706a388a79c0","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":954,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:11.771570733Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:52520"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:11.818889561Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"af0c5733-017b-4f58-83f4-b527da1d9520","remote_addr":"[fd00:1122:3344:101::a]:52520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47256,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:11.819000234Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:38551"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:11.822186956Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"36765044-cffd-412f-8d20-7c1da0b8ee53","remote_addr":"[fd00:1122:3344:101::a]:52520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2984,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:11.86583975Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b8312ca8-f0fc-4702-8945-326e7e41da58","remote_addr":"[fd00:1122:3344:101::b]:38551","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46745,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:11.866151458Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"9e1a65f7-3553-412b-8f10-49db1b95ad3e","remote_addr":"[fd00:1122:3344:101::b]:38551","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:12.137349335Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:41372"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:12.188137602Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"94f60f56-fd3f-4591-90b3-40c072f224e3","remote_addr":"[fd00:1122:3344:101::c]:41372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50710,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:12.188444099Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c59558b0-e8ca-4750-8c99-652909ea5a35","remote_addr":"[fd00:1122:3344:101::c]:41372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:13.677043758Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64608"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:13.677225712Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"7c21ef47-417d-4865-af69-9b83273cd0d0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64608","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":136,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:40.304039449Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"f7c98302-2cdd-456e-864b-23c9f283d27e","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":507,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:42.007030511Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41936"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:42.06008576Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b435881e-d431-4e45-a890-8835e9fce07c","remote_addr":"[fd00:1122:3344:101::b]:41936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":53012,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:42.060458938Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"82f6712c-2e52-47e8-9e60-f3d5b84c6936","remote_addr":"[fd00:1122:3344:101::b]:41936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:42.156368683Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:45733"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:42.202652677Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"620d65bb-7701-4772-a86b-8f5955ebe4a4","remote_addr":"[fd00:1122:3344:101::c]:45733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46227,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:42.202909283Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8e28c60f-d9c2-4167-b3a6-d8bc6ecfc719","remote_addr":"[fd00:1122:3344:101::c]:45733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:42.271179318Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:45806"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:42.318315792Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ed68dc1b-4bd0-4e6d-a5b7-a779d3d149a7","remote_addr":"[fd00:1122:3344:101::a]:45806","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46896,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:42.318558667Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"23d783c5-95ea-4992-827d-7e955f40cd8c","remote_addr":"[fd00:1122:3344:101::a]:45806","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:43.712285045Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36795"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:42:43.712405207Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"13b68c93-1ef2-4589-ae28-71ff2932d77e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36795","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":109,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:10.304982867Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"cff67190-23ce-44f1-8c81-7acb77e8e1bd","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1007,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:11.769517627Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41285"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:11.818079433Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e4785b62-2352-4cda-87f6-3b6b3adce374","remote_addr":"[fd00:1122:3344:101::b]:41285","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48497,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:11.818339709Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a5e4e685-ed6b-4156-8ae6-a6f10c381708","remote_addr":"[fd00:1122:3344:101::b]:41285","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:11.843479572Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:54362"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:11.886812819Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"3f9abf0b-cc09-4c2d-9cb3-d605c9dd1543","remote_addr":"[fd00:1122:3344:101::a]:54362","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43228,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:11.888141689Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"3a37b476-f8ba-436c-9eac-b3cd0f71cdb4","remote_addr":"[fd00:1122:3344:101::a]:54362","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1167,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:11.980903512Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:61086"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:12.026595013Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e72b7efd-3bbb-4705-a966-7a43ff34c793","remote_addr":"[fd00:1122:3344:101::c]:61086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45591,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:12.026861799Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"6cdfd779-befa-44f0-ad53-e6737ea0f867","remote_addr":"[fd00:1122:3344:101::c]:61086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:13.746841949Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33919"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:13.74692653Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"7c4f6835-14a6-447a-9f8d-270b1596a3a5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33919","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":79,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:40.304160074Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"f32969cb-f431-4fab-8488-51b1fc2a9d30","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":502,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:42.086016122Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:54678"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:42.136246586Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c86ec71b-e84d-4fea-98d5-025e06887d64","remote_addr":"[fd00:1122:3344:101::a]:54678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50177,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:42.136500212Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"5cb89f6e-2d4d-4bf1-a8c2-d21498970c8f","remote_addr":"[fd00:1122:3344:101::a]:54678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:42.218594492Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34729"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:42.265877679Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"407819fe-aece-47e3-81b3-cc0110b0018b","remote_addr":"[fd00:1122:3344:101::c]:34729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47181,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:42.266188036Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:53949"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:42.27029945Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"893ab7a3-470e-40b0-ac33-024e734cf8eb","remote_addr":"[fd00:1122:3344:101::c]:34729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4048,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:42.314416304Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6d7525c1-5068-4943-86c0-6b67d287a4e9","remote_addr":"[fd00:1122:3344:101::b]:53949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48057,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:42.315271924Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"51aee831-c5ba-4106-b11d-0f60edf5914b","remote_addr":"[fd00:1122:3344:101::b]:53949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":649,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:43.778991446Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45304"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:43:43.7791641Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"63984bac-1686-4063-951e-2d221a7eadf1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45304","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":137,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:10.304803442Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"1ee3547b-a772-4dee-bdb0-94683ed6e418","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":595,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:11.74552556Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:37620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:11.797602907Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c5544af7-f3c6-4cc5-bdc1-32ed04855d25","remote_addr":"[fd00:1122:3344:101::a]:37620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52025,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:11.797715459Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33093"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:11.841198809Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"4d1bf415-4e8a-48b1-9bdc-0959c2070a94","remote_addr":"[fd00:1122:3344:101::b]:33093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43422,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:11.841395274Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"741fa320-7cbc-4928-87c2-64fa9e76e5ea","remote_addr":"[fd00:1122:3344:101::a]:37620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:11.841607869Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"2aa780f7-7588-4384-abda-a48f3d001425","remote_addr":"[fd00:1122:3344:101::b]:33093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:11.985923876Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:33331"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:12.031068275Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ea883453-56d0-4893-93a5-514f45677450","remote_addr":"[fd00:1122:3344:101::c]:33331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45078,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:12.031416943Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"21174d92-220d-456b-89ac-c0cb62ae4b83","remote_addr":"[fd00:1122:3344:101::c]:33331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:13.812227667Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50448"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:13.812409362Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"a31c328e-47fa-4b60-921b-73938fc307e6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50448","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":146,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:31.049612596Z","hostname":"buskin","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:31.049700378Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:31.088879951Z","hostname":"buskin","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999995611s","next_cleanup":"Instant { tv_sec: 2018, tv_nsec: 724816193 }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:40.305307631Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"9bde673a-f9fd-4680-a3a4-6948d982ecb5","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":562,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:42.038971842Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33178"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:42.085895401Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"782a1220-7975-43c6-8b28-804f88b773a6","remote_addr":"[fd00:1122:3344:101::b]:33178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46863,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:42.086150107Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f55c2b62-f5c9-4a2e-922e-894149bdad33","remote_addr":"[fd00:1122:3344:101::b]:33178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:42.09902563Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47566"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:42.146739497Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"025ee2a4-c12b-4942-b3fd-7db656698044","remote_addr":"[fd00:1122:3344:101::a]:47566","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47615,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:42.146999843Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"227ee719-b90a-482b-8a26-d82e2f55881b","remote_addr":"[fd00:1122:3344:101::a]:47566","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:42.270919285Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51460"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:42.322003749Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"9478c000-f92b-4428-8db2-868799fbcf00","remote_addr":"[fd00:1122:3344:101::c]:51460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50977,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:42.322277385Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"93f46347-abc0-4efe-af0b-d975f2a9c8d6","remote_addr":"[fd00:1122:3344:101::c]:51460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:43.846601738Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44743"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:44:43.84669799Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"43795f93-0f8b-443f-a08a-127a12eed295","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44743","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:10.410538906Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"62744a3a-ac00-462e-997a-73daeb27b10d","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":858,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:11.759281569Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:46856"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:11.807266911Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"25ed4bc3-7d3a-4b96-a8a4-93c2f9c5755f","remote_addr":"[fd00:1122:3344:101::b]:46856","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47947,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:11.807549258Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7ad5f24d-c590-4361-83b8-86480db59240","remote_addr":"[fd00:1122:3344:101::b]:46856","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:11.851190272Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63514"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:11.893753702Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"cde72af7-2f71-4ca2-9ea0-7c0a4be12645","remote_addr":"[fd00:1122:3344:101::a]:63514","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":42471,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:11.894011918Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"2d74c934-36d8-4af2-88b5-ff96200fe454","remote_addr":"[fd00:1122:3344:101::a]:63514","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:12.062975006Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:57355"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:12.107684905Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"1092631c-fb19-4219-bc92-cabcd9eb308b","remote_addr":"[fd00:1122:3344:101::c]:57355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44618,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:12.108051473Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"ad589b53-e3a5-4df9-9c0e-7a9d2ea93577","remote_addr":"[fd00:1122:3344:101::c]:57355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:13.880745523Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36022"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:13.880904657Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"ea1862df-f310-4e74-9221-a51d2487b83b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36022","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.753263935Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41052"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.763101499Z","hostname":"buskin","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: 41914b1c-7c7c-4783-9ec3-871dd7abbc10, project_id: 1113ef06-5d8d-466a-b76c-b26a117dbb4e }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-29T22:45:26.242361Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(d7bd16c6-31ae-4e16-8b0b-39f708637820), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T22:45:26.300763528Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 85b9438f-2575-4707-9825-6040f72c0dc8, kind: Instance { id: 9883383c-c2d7-4d06-912d-01efc79f4813 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 13, 71])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4901523), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 85b9438f-2575-4707-9825-6040f72c0dc8, kind: Instance { id: 9883383c-c2d7-4d06-912d-01efc79f4813 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 13, 71])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4901523), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 85b9438f-2575-4707-9825-6040f72c0dc8, kind: Instance { id: 9883383c-c2d7-4d06-912d-01efc79f4813 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 13, 71])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4901523), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(4901523))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 85b9438f-2575-4707-9825-6040f72c0dc8, kind: Instance { id: 9883383c-c2d7-4d06-912d-01efc79f4813 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 13, 71])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4901523), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-4451298f773\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 4d389176-fb6d-413d-8e27-bd429348bc1a, read_only_parent: Some(Volume { block_size: 512, id: 5df86a88-af72-48f5-bc75-1dc4b6c744af, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 8dcd6415-5237-454e-90de-ca8b7b642105, key: Some(\"rgCq4Knna9PIDJMsjaZeYKmjIrKeHQjbGhgchywXXOg=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4d389176-fb6d-413d-8e27-bd429348bc1a, key: Some(\"65G0QO6u6gsVAVpYZNoE1rTO6h2cVjhXYgUKmYey1ZE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::15]:19003\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABHrbWdWJ1YAACttZ1YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAR621nVidWAAArbWdWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5ODgzMzgzYy1jMmQ3LTRkMDYtOTEyZC0wMWVmYzc5ZjQ4MTMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRDJVRHROR24yUTFlc0hQQ1FRNzJBbm11ZzhMeldNRGU5ckw5bHkvUXNsZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.763213292Z","hostname":"buskin","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.76358714Z","hostname":"buskin","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 85b9438f-2575-4707-9825-6040f72c0dc8, kind: Instance { id: 9883383c-c2d7-4d06-912d-01efc79f4813 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 13, 71])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4901523), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 85b9438f-2575-4707-9825-6040f72c0dc8, kind: Instance { id: 9883383c-c2d7-4d06-912d-01efc79f4813 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 13, 71])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4901523), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 85b9438f-2575-4707-9825-6040f72c0dc8, kind: Instance { id: 9883383c-c2d7-4d06-912d-01efc79f4813 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 13, 71])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4901523), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(4901523))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 85b9438f-2575-4707-9825-6040f72c0dc8, kind: Instance { id: 9883383c-c2d7-4d06-912d-01efc79f4813 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 13, 71])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4901523), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-4451298f773\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 4d389176-fb6d-413d-8e27-bd429348bc1a, read_only_parent: Some(Volume { block_size: 512, id: 5df86a88-af72-48f5-bc75-1dc4b6c744af, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 8dcd6415-5237-454e-90de-ca8b7b642105, key: Some(\"rgCq4Knna9PIDJMsjaZeYKmjIrKeHQjbGhgchywXXOg=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4d389176-fb6d-413d-8e27-bd429348bc1a, key: Some(\"65G0QO6u6gsVAVpYZNoE1rTO6h2cVjhXYgUKmYey1ZE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::15]:19003\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABHrbWdWJ1YAACttZ1YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAR621nVidWAAArbWdWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5ODgzMzgzYy1jMmQ3LTRkMDYtOTEyZC0wMWVmYzc5ZjQ4MTMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRDJVRHROR24yUTFlc0hQQ1FRNzJBbm11ZzhMeldNRGU5ckw5bHkvUXNsZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(d7bd16c6-31ae-4e16-8b0b-39f708637820), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T22:45:26.300763528Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-29T22:45:26.242361Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.764277656Z","hostname":"buskin","pid":1590,"uri":"/instances/9883383c-c2d7-4d06-912d-01efc79f4813","method":"PUT","req_id":"d06927fe-67da-4021-9606-15a870672a49","remote_addr":"[fd00:1122:3344:101::a]:41052","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":10976,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.86278651Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:45339"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.864168912Z","hostname":"buskin","pid":1590,"uri":"/instances/9883383c-c2d7-4d06-912d-01efc79f4813/state","method":"PUT","req_id":"589fd8df-f1e5-4c9a-a457-293b8ca84b9f","remote_addr":"[fd00:1122:3344:101::a]:45339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1269,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.884924844Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 253, 13, 71]), slot: 0, vni: Vni { inner: 4901523 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.885547539Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.919332178Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:26.940392118Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:28.994067558Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:28.994353135Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 29.181132ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:29.173512956Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 117.766845ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:29.211799238Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 291.023253ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:29.341024911Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 290.160196ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:29.647901202Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 1.028038328s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:29.948701144Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 976.427967ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:30.988201183Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 739.189577ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:31.975167635Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 593.751383ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:32.723957181Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 784.345748ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:33.328038462Z","hostname":"buskin","pid":1590,"zone":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.141725006Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 69.216796ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:34.150370733Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 85.903812ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:34.23013026Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 120.968969ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:34.326507495Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\") failed: Property not found. retry in 299.119517ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:34.456767593Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.764754888Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.799674974Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABHrbWdWJ1YAACttZ1YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAR621nVidWAAArbWdWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5ODgzMzgzYy1jMmQ3LTRkMDYtOTEyZC0wMWVmYzc5ZjQ4MTMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRDJVRHROR24yUTFlc0hQQ1FRNzJBbm11ZzhMeldNRGU5ckw5bHkvUXNsZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-4451298f773\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 4d389176-fb6d-413d-8e27-bd429348bc1a, read_only_parent: Some(Volume { block_size: 512, id: 5df86a88-af72-48f5-bc75-1dc4b6c744af, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 8dcd6415-5237-454e-90de-ca8b7b642105, key: Some(\"rgCq4Knna9PIDJMsjaZeYKmjIrKeHQjbGhgchywXXOg=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4d389176-fb6d-413d-8e27-bd429348bc1a, key: Some(\"65G0QO6u6gsVAVpYZNoE1rTO6h2cVjhXYgUKmYey1ZE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::15]:19003\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 9883383c-c2d7-4d06-912d-01efc79f4813, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 1113ef06-5d8d-466a-b76c-b26a117dbb4e, silo_id: 41914b1c-7c7c-4783-9ec3-871dd7abbc10 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.801484275Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.910919948Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-04-29T22:45:34.912417692Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.912440482Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.912447013Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-29T22:45:34.912417692Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(d7bd16c6-31ae-4e16-8b0b-39f708637820), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T22:45:26.300763528Z }","propolis_id":"d7bd16c6-31ae-4e16-8b0b-39f708637820"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.912453593Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d7bd16c6-31ae-4e16-8b0b-39f708637820), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T22:45:26.300763528Z }, propolis_id: d7bd16c6-31ae-4e16-8b0b-39f708637820, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-29T22:45:34.912417692Z } }","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-04-29T22:45:34.944485172Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.944502483Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.944517693Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-29T22:45:34.944485172Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(d7bd16c6-31ae-4e16-8b0b-39f708637820), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T22:45:26.300763528Z }","propolis_id":"d7bd16c6-31ae-4e16-8b0b-39f708637820"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:34.944525983Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d7bd16c6-31ae-4e16-8b0b-39f708637820), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T22:45:26.300763528Z }, propolis_id: d7bd16c6-31ae-4e16-8b0b-39f708637820, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-29T22:45:34.944485172Z } }","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:40.410121864Z","hostname":"buskin","pid":1590,"uri":"//ce40678a-3256-41be-b175-a8516c162f00","method":"GET","req_id":"76237a10-8325-4389-9c5d-4511df7eddd6","remote_addr":"[fd00:1122:3344:101::d]:45107","local_addr":"[fd00:1122:3344:101::1]:46822","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"ce40678a-3256-41be-b175-a8516c162f00","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":690,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:42.058651425Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37053"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:42.104109981Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d087d4a6-e304-4804-9a3e-bcb0e48bd257","remote_addr":"[fd00:1122:3344:101::b]:37053","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45373,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:42.104250354Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:48619"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:42.107949588Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8828f995-3761-43ea-8137-e2181c2d6829","remote_addr":"[fd00:1122:3344:101::b]:37053","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3477,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:42.145068593Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"73078ea5-60d9-48e9-9559-aee8a60e08db","remote_addr":"[fd00:1122:3344:101::a]:48619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":40694,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:42.145407381Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"03aa2fde-2f5d-4c43-aa9c-b540d060ced2","remote_addr":"[fd00:1122:3344:101::a]:48619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:42.201138021Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:42331"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:42.239809562Z","hostname":"buskin","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a6b0fdd1-3fd5-4148-a7cb-b3eceb1d42bc","remote_addr":"[fd00:1122:3344:101::c]:42331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":38611,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:42.240033597Z","hostname":"buskin","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"5ce82930-82b7-40a2-9b91-ac4b08d12954","remote_addr":"[fd00:1122:3344:101::c]:42331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:43.913084067Z","hostname":"buskin","pid":1590,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56784"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:43.913179149Z","hostname":"buskin","pid":1590,"uri":"/baseboard","method":"GET","req_id":"c83d8a4c-783b-465e-bd4e-41c9ef9dd47b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56784","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":70,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.278711216Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:52841"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.279555485Z","hostname":"buskin","pid":1590,"uri":"/instances/9883383c-c2d7-4d06-912d-01efc79f4813/state","method":"PUT","req_id":"6265b188-406b-46e4-98d0-636109113cc9","remote_addr":"[fd00:1122:3344:101::a]:52841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":739,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-04-29T22:45:44.279737329Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.27979232Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.279851512Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-29T22:45:44.279737329Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(d7bd16c6-31ae-4e16-8b0b-39f708637820), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T22:45:26.300763528Z }","propolis_id":"d7bd16c6-31ae-4e16-8b0b-39f708637820"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.279891033Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d7bd16c6-31ae-4e16-8b0b-39f708637820), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T22:45:26.300763528Z }, propolis_id: d7bd16c6-31ae-4e16-8b0b-39f708637820, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-29T22:45:44.279737329Z } }","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-04-29T22:45:44.299147681Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.299169132Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.299188052Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-04-29T22:45:44.299147681Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-29T22:45:44.299147681Z }","propolis_id":"d7bd16c6-31ae-4e16-8b0b-39f708637820"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.299211733Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.299222223Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.299454948Z","hostname":"buskin","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.29951571Z","hostname":"buskin","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/2a8df99a-fa8f-45a4-9b0b-dc55770b6483/debug/bundle/zone\", \"/pool/int/dee1c314-6799-4686-a21f-d577311c1a7d/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/9b351946-585e-4029-8f1b-9f69550dce71/crypt/debug\", \"/pool/ext/478a6e43-eea9-40ca-b6f4-6a492473e177/crypt/debug\", \"/pool/ext/e3118907-da30-4e85-a767-c5a42a2e46ca/crypt/debug\", \"/pool/ext/d280116a-a989-463f-b7e1-373c7fcb5463/crypt/debug\", \"/pool/ext/661f9fd6-5e25-45d3-955c-06a9af9dad35/crypt/debug\", \"/pool/ext/9962603d-6bc6-41a3-a346-9382c9e6bfb4/crypt/debug\", \"/pool/ext/ff581678-2bea-470b-9448-890977b6d596/crypt/debug\", \"/pool/ext/4b1821a9-dc17-4642-9053-2192b3ae698e/crypt/debug\", \"/pool/ext/3c2c8854-3269-4025-ae74-138b796a9368/crypt/debug\"] }","zone_name":"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:44.91849626Z","hostname":"buskin","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820\", bundle_id: 0d9d9fbe-f874-4f82-a412-9e973325b535 }, time_created: 2024-04-29T22:45:44.300812809Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_d7bd16c6-31ae-4e16-8b0b-39f708637820","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:44.918597992Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:46.679490703Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:46.701150186Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-29T22:45:44.299147681Z }, propolis_id: d7bd16c6-31ae-4e16-8b0b-39f708637820, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-29T22:45:46.701087494Z } }","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:46.797757207Z","hostname":"buskin","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55818"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T22:45:46.798850812Z","hostname":"buskin","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:46.798872382Z","hostname":"buskin","pid":1590,"uri":"/v2p/85b9438f-2575-4707-9825-6040f72c0dc8","method":"DELETE","req_id":"c29f9ed1-3d2b-4cf3-8d10-c7690820e03c","remote_addr":"[fd00:1122:3344:101::a]:55818","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":963,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:47.023338775Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-29T22:45:44.299147681Z }, propolis_id: d7bd16c6-31ae-4e16-8b0b-39f708637820, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-29T22:45:46.701087494Z } }","instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T22:45:47.026985838Z","hostname":"buskin","pid":1590,"instance_id":"9883383c-c2d7-4d06-912d-01efc79f4813","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"}